Source file benchmark.ml

(*****************************************************************************)
(*                                                                           *)
(* Open Source License                                                       *)
(* Copyright (c) 2020 Nomadic Labs. <contact@nomadic-labs.com>               *)
(*                                                                           *)
(* Permission is hereby granted, free of charge, to any person obtaining a   *)
(* copy of this software and associated documentation files (the "Software"),*)
(* to deal in the Software without restriction, including without limitation *)
(* the rights to use, copy, modify, merge, publish, distribute, sublicense,  *)
(* and/or sell copies of the Software, and to permit persons to whom the     *)
(* Software is furnished to do so, subject to the following conditions:      *)
(*                                                                           *)
(* The above copyright notice and this permission notice shall be included   *)
(* in all copies or substantial portions of the Software.                    *)
(*                                                                           *)
(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*)
(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,  *)
(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL   *)
(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*)
(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING   *)
(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER       *)
(* DEALINGS IN THE SOFTWARE.                                                 *)
(*                                                                           *)
(*****************************************************************************)

type group = Standalone | Group of string | Generic

type 'config parameters = {bench_number : int; config : 'config}

type purpose = Other_purpose of string | Generate_code of string

let pp_purpose ppf t =
  match t with
  | Other_purpose s -> Format.fprintf ppf "Other purpose: %s" s
  | Generate_code s -> Format.fprintf ppf "Generate code: %s" s

module type Benchmark_base = sig
  val name : Namespace.t

  val info : string

  val module_filename : string

  val purpose : purpose

  val tags : string list

  type config

  val default_config : config

  val config_encoding : config Data_encoding.t

  type workload

  val workload_encoding : workload Data_encoding.t

  val workload_to_vector : workload -> Sparse_vec.String.t
end

module type S = sig
  include Benchmark_base

  val models : (string * workload Model.t) list

  include Generator.S with type config := config and type workload := workload
end

module type Simple = sig
  include Benchmark_base

  val group : group

  val model : workload Model.t

  val create_benchmark :
    rng_state:Random.State.t -> config -> workload Generator.benchmark
end

module type Simple_with_num = sig
  include Benchmark_base

  val group : group

  val model : workload Model.t

  include Generator.S with type workload := workload and type config := config
end

type t = (module S)

type simple = (module Simple)

type simple_with_num = (module Simple_with_num)

let pp ppf (module Bench : S) =
  let open Bench in
  let open Format in
  let f fmt = fprintf ppf fmt in
  let pp_config fmt config =
    Data_encoding.Json.pp fmt
    @@ Data_encoding.Json.construct config_encoding config
  in
  f "@[<v>" ;
  f "name: %a@;" Namespace.pp name ;
  f "info: %s@;" info ;
  f "module_filename: %s@;" module_filename ;
  f "purpose: %a@;" pp_purpose purpose ;
  f
    "tags: [%a]@;"
    (pp_print_list ~pp_sep:(fun ppf () -> fprintf ppf "; ") pp_print_string)
    tags ;
  f "@[<v2>default_config:@ @[%a@]@]@;" pp_config default_config ;
  f
    "@[<v2>local models for inference:@ @[<v>%a@]@]@;"
    (pp_print_list (fun ppf (local_model_name, model) ->
         fprintf ppf "@[<v2>%s:@ @[%a@]@]" local_model_name Model.pp model))
    models ;
  f "@]"

type ('cfg, 'workload) poly =
  (module S with type config = 'cfg and type workload = 'workload)

type packed = Ex : ('cfg, 'workload) poly -> packed

let name ((module B) : t) = B.name

let info ((module B) : t) = B.info

let tags ((module B) : t) = B.tags

let ex_unpack : t -> packed = fun (module Bench) -> Ex ((module Bench) : _ poly)

let get_free_variable_set (module Bench : S) =
  List.fold_left
    (fun acc (_, model) ->
      Free_variable.Set.union acc @@ Model.get_free_variable_set_of_t model)
    Free_variable.Set.empty
    Bench.models