Youtube-ML

I just dug this one out from the past (beginning of 2008 I think) and I figured I might as well throw it in here along with all the others.

This is a script for giving youtube-dl a whole list of download links instead of doing one-by-one. Nothing to it really.

So first, if you download this stuff, you probably want to compile it with OCaml like this:

ocaml -o youtube-ml str.cma youtube.ml

(Use the str module, and compile youtube.ml as youtube-ml.)

Now that you have the binary executable you need to create a file containing the addresses you want to download and the names of files you want to save those downloads as. These will be separated by a three-hyphen delimiter. Example:

Beat the Horse.mp4 --- http://www.youtube.com/watch?v=8oJgqbgvInk
Pas Encore.mp4 --- http://www.youtube.com/watch?v=FCCoggBkcdM
Centrifuge.mp4 --- http://www.youtube.com/watch?v=C_Gy4vzS4U8
Hail Mary.mp4 --- http://www.youtube.com/watch?v=fYy2p_0DVMU

And then you run the program to start the process, like so:

youtube-ml --best-quality FILE

In this case the files will be downloaded in best quality (as mp4) as well.

I might add that you need youtube-dl installed, because this script does nothing except reading from a file and passing the information to youtube-dl.

The code:

 
  1 (** 
  2  * Youtube-ML 
  3  * 
  4  * Use youtube-dl to download videos from a list saved in a file.  
  5  * Sort of batch-process youtube-dl downloads. 
  6  * 
  7  * Compiling: 
  8  *  ocamlc -o youtube-ml str.cma youtube.ml 
  9  * 
 10  * Example usage: 
 11  *  Download using a file in best quality: 
 12  *      ./youtube-ml --best-quality list_of_files.txt 
 13  *  Download using a bash ad-hoc file and in quiet mode: 
 14  *  ./youtube-ml --quiet <<$ 
 15  *      some_filename.flv --- http://some_address/gibberish 
 16  *      other_filename.flv --- http://other_address/gibberish 
 17  *  $ 
 18  * 
 19  * Parameters: 
 20  *  --help - show usage information, 
 21  *  --quiet - do not display additional information messages, 
 22  *  --best-quality - download in MP4 instead of FLV format. 
 23  * 
 24  * Requires: 
 25  *  youtube-dl (http://bitbucket.org/rg3/youtube-dl/wiki/Home) 
 26  * 
 27  * Author: 
 28  *  Konrad Siek 
 29  * 
 30  * License: 
 31  *  Copyright 2010 Konrad Siek  
 32  * 
 33  *  This program is free software: you can redistribute it and/or modify 
 34  *  it under the terms of the GNU General Public License as published by 
 35  *  the Free Software Foundation, either version 3 of the License, or 
 36  *  (at your option) any later version. 
 37  * 
 38  *  This program is distributed in the hope that it will be useful, 
 39  *  but WITHOUT ANY WARRANTY; without even the implied warranty of 
 40  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 41  *  GNU General Public License for more details. 
 42  * 
 43  *  You should have received a copy of the GNU General Public License 
 44  *  along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 45  *) 
 46 open Str;; 
 47  
 48 let youtube_cmd = "youtube-dl";; 
 49 let delimiter = "[\t ]+---[\t ]+";; 
 50 let best_quality_flag = ref false;; 
 51 let quiet_flag = ref false;; 
 52  
 53 exception InvalidString of string ;; 
 54  
 55 type filename = string;; 
 56 type address = string;; 
 57 type clip = filename * address;; 
 58  
 59 (* Convert line into a tupple of type clip. *) 
 60 let clip_of_string line = 
 61     let regex = Str.regexp delimiter in 
 62     let result = Str.split regex line in 
 63     match result with 
 64     | file :: http :: [] -> (file, http) 
 65     | _ -> raise (InvalidString ("Line '" ^ line ^ "' is invalid.")) 
 66 ;; 
 67  
 68 (* Read a file and break it down into tupples. *) 
 69 let read_file filename = 
 70     let channel = open_in filename in 
 71     let rec read channel result = 
 72     try 
 73         let line = input_line channel  in 
 74         let clip = clip_of_string line in 
 75         let result = result @ [clip] in 
 76         read channel result
 77 (* with Invalid_string -> raise Invalid_string *) 
 78     with 
 79         | InvalidString message -> raise (InvalidString message) 
 80         | _ -> result
 81     in 
 82     read channel [] 
 83 ;; 
 84  
 85 (* Print completed message. *) 
 86 let print_completed output = 
 87     if not !quiet_flag then 
 88         print_endline ("Download into file '" ^ output ^ "' completed.") 
 89 ;; 
 90  
 91 (* Print incomplete message. *) 
 92 let print_incomplete output = 
 93     if not !quiet_flag then 
 94         print_endline ("Download into file '" ^ output ^ "' problematic.") 
 95 ;; 
 96  
 97 (* Print that a file is being downloaded. *) 
 98 let print_intro http output = 
 99     if not !quiet_flag then 
100         print_endline ("Downloading " ^ http ^ " as " ^ output) 
101 ;; 
102  
103 (* Download clips listed in one file. *) 
104 let download filename = 
105     let clips = read_file filename in 
106     let download (output, http) = 
107         let command = if !best_quality_flag then 
108             youtube_cmd ^ " -b -o \"" ^ output ^ "\" -q \"" ^ http ^ "\"" 
109         else 
110             youtube_cmd ^ " -o \"" ^ output ^ "\" -q \"" ^ http ^ "\"" in 
111         print_intro http output; 
112         let result = Sys.command command in 
113         if result = 0 then 
114             print_completed output
115         else 
116             print_incomplete output
117     in 
118     List.iter download clips
119 ;; 
120  
121 (* Print message on how to use the program.  *) 
122 let print_usage program_name = 
123     let usage = 
124       "Usage: \n" ^ 
125       "\t" ^ program_name ^ " " ^ "filename [filename ...] \n" ^ 
126       "\t" ^ program_name ^ " " ^ "--best-quality filename [filename ...] \n" ^ 
127       "\t" ^ program_name ^ " " ^ "--help \n" ^ 
128       "Files: \n" ^ 
129       "\t" ^ "The files should consist of lines, each line composed of:" ^ 
130       "\t\t" ^ " - the file to which save the song," ^ 
131       "\t\t" ^ " - the delimiter consisting of three hyphens ('---')" ^ 
132       "\t\t" ^ "   which can be surrounded by tabs and spaces," ^ 
133       "\t\t" ^ " - address of the song." 
134     in 
135     print_endline usage
136 ;; 
137  
138 (* Print message that given file does not exist. *) 
139 let print_file_not_exists file = 
140     let message = "File '" ^ file ^ "' does not exist. " in 
141     print_endline message
142 ;; 
143  
144 (* Process all arguments and conduct appropriate action. *) 
145 let rec process_args program args = 
146     match args with 
147     | arg :: tail -> 
148         if String.length arg > 1 && String.get arg 0 = '-' then 
149             match arg with 
150             | "--help" -> print_usage program
151             | "--best-quality" -> 
152                 best_quality_flag := true; 
153                 process_args program tail
154             | "--quiet" -> 
155                 quiet_flag := true; 
156                 process_args program tail
157             | _ -> print_usage program
158         else 
159             download arg
160     | [] -> () 
161 ;; 
162  
163 (* Read command-line arguments and launch operations. *) 
164 let main () = 
165 let length = Array.length Sys.argv in 
166 let program = Sys.argv.(0) in 
167     if length > 1 then 
168         process_args program (List.tl (Array.to_list Sys.argv)) 
169     else 
170         print_usage program
171 ;; 
172  
173 main () 

The code is also available at GitHub as ocaml/youtube.ml.

Substitute

Another Ocaml script (although, I suppose it isn't a script, because I compile it). I really like to write in Ocaml, because it's a different kind of thought process.

This one is just a little tool that I sometimes like to use, when I have to go through a file and change the name of some array or similar. Granted, this can be done automatically through many text editors, but sometimes it's a hassle to even open a file, especially if it's biggish.

Besides, I found this bugger on my desktop today, in a slightly different form, and couldn't figure out what it was, that I wanted it to do (which is embarrassing, because it wasn't that hard). When I did figure out what it does, I decided to complicate it a bit, and use some of that fantastic functional paradigm, that Ocaml offers (lines 81, 84, and others).

I used a trick here, that I often use when doing functional programming. It's in the function on line 94 - handle_arguments. Inside this function I create a function with the same name. I can do this, because, unless I use the rec keyword during declaration, the function is not declared yet within itself. The inner function is recursive (the rec keyword), so that it can traverse a list and pass on the results, whilst the outer function calls it with proper arguments. Thanks to this, I don't have to remember to call handle_arguments with an additional parameter (an empty list for the results) from the outside, so the code is just that little bit cleaner.

Technical hints:
To compile the code just go:

ocamlc -o substitute str.cma substitute.ml

As you can see, it needs the additional Str module.

To run the program go:

ocamlc -o substitute str.cma substitute.ml

Oh yeah, and if you need a different format for the command line arguments, you can change the delimiter at line 28.

Warning: The substitution may fail somewhere, so if you try doing something important with this piece of software, better have a backup copy of whathever you're using it with.

The program:

1  (**
2   * Substitute
3   * Substitutes things for other things in files.
4   *
5   * Compiling:
6   *  ocamlc -o substitute str.cma substitute.ml
7   * Example usage:
8   *  echo '$0 $1!' | ./substitute '$0->Hello' '$1->world'
9   *  #=> Hello world!
10  * Potential issues:
11  *  May substitute more than you'd like, so keep a 
12  *  backup.
13  * Parameters:
14  *  Each parameter is in the form of a placeholder,
15  *  the delimiter '->' and a substitution
16  *  (best to look at the example).
17  * Requires:
18  *  Ocaml (http://caml.inria.fr/)
19  * Author:
20  *  Konrad Siek
21  *)
22 
23 (**
24  * Delimiter for program arguments.
25  * Establishes which part of the argument is the 
26  * placeholder and which is the substitution.
27  *)
28 let delimiter = "->";;
29 (**
30  * Substitutes a placeholder with a phrase.
31  * This function runs recursivelly on a list.
32  * @param what - placeholder
33  * @param into - substitution
34  * @param contents - array of strings to traverse
35  * @return an array of strings
36  *)
37 let substitute functions contents =
38     let rec apply functions content =
39         match functions with
40         | transform::tail -> 
41             apply tail (transform content)
42         | [] -> content
43     in
44     let result = ref [] in
45     let iterate element =
46         result := !result @ 
47         [apply functions element]
48     in
49     List.iter iterate contents;
50     !result
51 ;;
52 (**
53  * Outputs the contents of an array to standard
54  * output.
55  * @param contents - an array of strings
56  *)
57 let rec print_contents contents = 
58     match contents with
59     | head::tail -> 
60         print_endline head; 
61         print_contents tail
62     | [] -> () 
63 ;;
64 (**
65  * Converts a program argument into a translation
66  * function.
67  * @param argument
68  * @return a function
69  *)
70 let handle_argument argument = 
71     let regex = Str.regexp_string delimiter in
72     let bits = Str.split regex argument in 
73     if List.length bits < 2 then (
74         prerr_string 
75             ("Illegal argument: '" ^ argument ^ "'");
76         prerr_newline ();
77         fun input -> input
78     ) else (
79         let from = Str.regexp_string (List.hd bits) in 
80         let into = List.fold_left
81             (fun a b -> a ^ delimiter ^ b) 
82             (List.hd (List.tl bits)) 
83             (List.tl (List.tl bits)) in
84         fun input -> 
85             (Str.global_replace from into input)
86     )
87 ;;
88 (**
89  * Converts a list of program arguments into a
90  * list of translation functions.
91  * @param arguments
92  * @return functions
93  *)
94 let handle_arguments arguments = 
95     let rec handle_arguments arguments results = 
96         match arguments with
97         | head::tail -> 
98             let argument = 
99                 (handle_argument head) in 
100            handle_arguments tail
101                (results @ [argument])
102        | [] -> results
103    in
104    handle_arguments arguments []
105;;
106(**
107 * Grab input from standard input - read until
108 * an end of stream occurs.
109 * @param unit
110 * @return list of strings
111 *)
112let read_input () =
113    let list = ref [] in 
114    let _ = try 
115        while true do
116            let line = input_line stdin in
117            list := !list @ [line]
118        done
119    with _ -> () in 
120    !list
121;;    
122
123(* Convert argument vector to list *)
124let arguments = (List.tl (Array.to_list Sys.argv)) in
125(* Translate arguments to functions *)
126let functions = handle_arguments arguments in
127(* Read contents from standard input *)
128let contents = read_input () in 
129(* Apply transformations on contents *)
130let results = substitute functions contents in
131(* And print results to standard output *)
132print_contents results;;

Dada generator

Disclaimer: this program is not useful in any way.

Ok, I wrote this one when I was listening to a lecture on databases and data warehouses - I was exploring functional programming at the time, and decided that if you really thought about it you could make a very nice and pleasant function that does almost absolutely nothing, apart from being aesthetically pleasing. After I wrote one on paper, I decided that it could be expanded a bit, but it was too much hassle to do by hand... so, a generator!

What this program does is actually generate a program, which, upon running will recursively pass all its parameters to itself, and when the recursion stops it will use the first of the parameters as a function, supplying it with the other parameters as parameters... is this clear? I suppose an example is in order...

Ok, so first, I run the program in the following way:

ocaml dada_gen.ml 5 f a b c > dada_ex.ml

...and got the following code as a result (in the file dada_ex.ml):

1 let f a b c =
2    let f a b c =
3       let f a b c =
4          let f a b c =
5             let f a b c =
6                let f a b c =
7                   a b c 
8                in f a b c 
9             in f a b c 
10         in f a b c 
11      in f a b c 
12   in f a b c 
13;;
14

Ok, then I run the Ocaml top-level like this:

ocaml

Inside the toplevel I loaded the file:

#use "dada_ex.ml";;

...and got the response:

val f : ('a -> 'b -> 'c) -> 'a -> 'b -> 'c = <fun>

...and then the function f funtion like this:

f (+) 2 2

...and got in return:

- : int = 4

...which just means that the result is 4. I could've achieved the same by going:

(+) 2 2;;

...or even

2 + 2

That's why its dada. It's not meant to be useful!

Yeah, I know, I'm weird.

Here's the script:

1 (**
2  * Dada generator
3  * Generates a completely useless but aesthetically pleasing dada program
4  * 
5  * Potential issues:
6  *  None
7  * Parameters:
8  *  number of internal functions
9  *  name of the dadaistic function
10 *  list of passed parameters
11 * Requires:
12 *  Ocaml (http://caml.inria.fr/)
13 *  Sense of humor
14 *)
15
16(**
17 * Function responsible for generating dadaistic programs basing on the 
18 * supplied parameters
19 * @param function_name the name of the main function
20 * @param parameter_list a list of strings which will be inserted as the 
21 *  body of the inner-most function
22 * @param nested_function_count the number of functions generated inside 
23 *  the main function
24 * @author Konrad Siek
25 *)
26let generate_dada function_name parameter_list nested_function_count =
27   let rec list_to_string list =
28      match list with 
29        | head::tail -> head ^ " " ^ (list_to_string tail)
30        | [] -> ""
31   in
32   let rec tabs count = 
33      if count = 0 then
34         ""
35      else
36         "   " ^ (tabs (count - 1))
37   in
38   let program = ref ("let " ^ function_name  ^ " " ^ 
39        (list_to_string parameter_list) ^ "=\n") in
40   let _ = for i = 1 to nested_function_count do
41     program := !program ^ (tabs i);
42     program := !program ^ "let " ^ function_name ^ " " ^ 
43        (list_to_string parameter_list) ^ "=\n"
44   done in
45   program := !program ^ (tabs (nested_function_count + 1)) ^ 
46        (list_to_string parameter_list) ^ "\n";
47   let _ = for i = nested_function_count downto 1 do
48     program := !program ^ (tabs i);
49     program := !program ^ "in " ^  function_name ^ " " ^ 
50        (list_to_string parameter_list) ^ "\n"
51   done in
52   program := !program ^ ";;\n";
53   !program
54;;
55
56(* Main method calling the generation function. *)
57if Array.length Sys.argv > 3 then 
58   let count = int_of_string Sys.argv.(1) in
59   let function_name = Sys.argv.(2) in
60   let parameters = Array.to_list 
61    (Array.sub Sys.argv 3 (Array.length Sys.argv - 3)) in
62   let program = generate_dada function_name parameters count in
63   print_endline program
64else
65   print_endline ("Usage:\n\t" ^ Sys.argv.(0) ^ " <function_count> " ^ 
66        "<main_function_name> <parameter> [ ... <parameter> ]")
67;;

The code is also available at GitHub as ocaml/dada_generator.ml.