package owl-base

  1. Overview
  2. Docs
package owl-base
Legend:
Page
Library
Module
Module type
Parameter
Class
Class type
Source

Source file owl_io.ml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
# 1 "src/base/misc/owl_io.ml"
(*
 * OWL - OCaml Scientific and Engineering Computing
 * Copyright (c) 2016-2020 Liang Wang <liang.wang@cl.cam.ac.uk>
 *)

open Owl_utils

(* read a file of a given path *)
let read_file ?(trim = true) f =
  let h = open_in f in
  Fun.protect
    (fun () ->
      let s = Stack.make () in
      (try
         while true do
           let l =
             match trim with
             | true  -> input_line h |> String.trim
             | false -> input_line h
           in
           Stack.push s l
         done
       with
      | End_of_file -> ());
      Stack.to_array s)
    ~finally:(fun () -> close_in h)


let read_file_string f =
  let ic = open_in f in
  Fun.protect
    (fun () ->
      let n = in_channel_length ic in
      let s = Bytes.create n in
      really_input ic s 0 n;
      Bytes.to_string s)
    ~finally:(fun () -> close_in ic)


(* write a file of a given path *)
let write_file ?(_flag = Open_creat) f s =
  let h = open_out f in
  Fun.protect (fun () -> Printf.fprintf h "%s" s) ~finally:(fun () -> close_out h)


(* iterate every doc in the corpus without loading the whole corpus in the
  memory, then apply passed in function f. note that each line is a doc.
 *)
let iteri_lines_of_file ?(verbose = true) f fname =
  let i = ref 0 in
  let h = open_in fname in
  Fun.protect
    (fun () ->
      let t0 = Unix.gettimeofday () in
      let t1 = ref (Unix.gettimeofday ()) in
      try
        while true do
          f !i (input_line h);
          i := !i + 1;
          (* output summary if in verbose mode *)
          if verbose = true
          then (
            let t2 = Unix.gettimeofday () in
            if t2 -. !t1 > 5.
            then (
              t1 := t2;
              let speed = float_of_int !i /. (t2 -. t0) |> int_of_float in
              Owl_log.info "processed %i, avg. %i docs/s" !i speed))
        done
      with
      | End_of_file -> ())
    ~finally:(fun () -> close_in h)


(* map every doc in the corpus into another type *)
let mapi_lines_of_file f fname =
  let stack = Owl_utils.Stack.make () in
  iteri_lines_of_file (fun i s -> Owl_utils.Stack.push stack (f i s)) fname;
  Owl_utils.Stack.to_array stack


(* similar to iteri_lines_of_file but for marshaled file *)
let iteri_lines_of_marshal ?(verbose = true) f fname =
  let i = ref 0 in
  let h = open_in_bin fname in
  Fun.protect
    (fun () ->
      let t1 = ref (Unix.gettimeofday ()) in
      let i1 = ref 0 in
      try
        while true do
          f !i (Marshal.from_channel h);
          i := !i + 1;
          (* output summary if in verbose mode *)
          if verbose = true
          then (
            let t2 = Unix.gettimeofday () in
            if t2 -. !t1 > 5.
            then (
              let speed = float_of_int (!i - !i1) /. (t2 -. !t1) |> int_of_float in
              i1 := !i;
              t1 := t2;
              Owl_log.info "processed %i, avg. %i docs/s" !i speed))
        done
      with
      | End_of_file -> ())
    ~finally:(fun () -> close_in h)


(* similar to mapi_lines_of_file but for marshaled file *)
let mapi_lines_of_marshal f fname =
  let stack = Owl_utils.Stack.make () in
  iteri_lines_of_marshal (fun i s -> Owl_utils.Stack.push stack (f i s)) fname;
  Owl_utils.Stack.to_array stack


(* save a marshalled object to a file *)
let marshal_to_file ?(flags = []) x f =
  let h = open_out_bin f in
  Fun.protect (fun () -> Marshal.to_channel h x flags) ~finally:(fun () -> close_out h)


(* load a marshalled object from a file *)
let marshal_from_file f =
  let h = open_in_bin f in
  Fun.protect
    (fun () ->
      let s = really_input_string h (in_channel_length h) in
      Marshal.from_string s 0)
    ~finally:(fun () -> close_in h)


let head n fname =
  let lines = Owl_utils.Stack.make () in
  (try
     iteri_lines_of_file
       (fun i s ->
         assert (i < n);
         Owl_utils.Stack.push lines s)
       fname
   with
  | exn -> Owl_log.warn "Owl_io.head: ignored exception %s" (Printexc.to_string exn));
  Owl_utils.Stack.to_array lines


(* TODO *)
let _tail _n _fname = raise (Owl_exception.NOT_IMPLEMENTED "owl_io._tail")

let read_csv ?(sep = '\t') fname =
  let lines = Owl_utils.Stack.make () in
  iteri_lines_of_file
    (fun _i s ->
      String.trim s
      |> String.split_on_char sep
      |> Array.of_list
      |> Owl_utils.Stack.push lines)
    fname;
  Owl_utils.Stack.to_array lines


let write_csv ?(sep = '\t') x fname =
  let h = open_out fname in
  Fun.protect
    (fun () ->
      Array.iter
        (fun row ->
          let s =
            Array.fold_left (fun acc elt -> Printf.sprintf "%s%s%c" acc elt sep) "" row
          in
          Printf.fprintf h "%s\n" s)
        x)
    ~finally:(fun () -> close_out h)


let read_csv_proc ?(sep = '\t') proc fname =
  iteri_lines_of_file
    (fun i s -> String.trim s |> String.split_on_char sep |> Array.of_list |> proc i)
    fname


let write_csv_proc ?(sep = '\t') x proc fname =
  let h = open_out fname in
  Fun.protect
    (fun () ->
      Array.iter
        (fun row ->
          let s =
            Array.fold_left
              (fun acc elt -> Printf.sprintf "%s%s%c" acc (proc elt) sep)
              ""
              row
          in
          Printf.fprintf h "%s\n" s)
        x)
    ~finally:(fun () -> close_out h)


let csv_head ?(sep = '\t') idx fname =
  let h = open_in fname in
  Fun.protect
    (fun () ->
      for _i = 1 to idx - 1 do
        input_line h |> ignore
      done;
      input_line h |> String.trim |> String.split_on_char sep |> Array.of_list)
    ~finally:(fun () -> close_in h)
OCaml

Innovation. Community. Security.

GitHub Discord Twitter Peertube RSS
About
Changelog Releases Industrial Users Academic Users Why OCaml
Ecosystem
Packages Community Events OCaml Planet Jobs
Resources
Install OCaml Get Started Platform Tools Language Manual Standard Library API Books Exercises Papers OCaml Playground Logo
Policies
Carbon Footprint Governance Privacy Code of Conduct