Merge branch 'compatible' into dkijania/merge/compatible_into_develop_020425

Author: dkijania

automation/README.md

@@ -264,11 +264,11 @@ There are several public Grafana dashboards available here:
 
 The purpose of a public testnet is to allow end-users to try out the software and learn how to operate it. Thus, we accept sign-ups for stake to be allocated in the genesis, and commit those keys to the compiled genesis ledger.
 
-For context, these keys correspond to the "Fish Keys" in the QA Net deployments, and Online Fish Keys are ommitted in a Public Testnet deployment and "Offline Fish Keys" are instead delegated to the submitted User Keys.
+For context, these keys correspond to the "Fish Keys" in the QA Net deployments, and Online Fish Keys are omitted in a Public Testnet deployment and "Offline Fish Keys" are instead delegated to the submitted User Keys.
 
 ### Generate Genesis Ledger
 
-Once you have the keys for your deploymenet created, and the Staker Keys saved to a CSV, you can use them to generate a genesis ledger with the following command.
+Once you have the keys for your deployment created, and the Staker Keys saved to a CSV, you can use them to generate a genesis ledger with the following command.
 
 ```
 scripts/generate-keys-and-ledger.sh

buildkite/src/Command/Bench/Base.dhall

@@ -1,3 +1,5 @@
+let B = ../../External/Buildkite.dhall
+
 let PipelineMode = ../../Pipeline/Mode.dhall
 
 let PipelineTag = ../../Pipeline/Tag.dhall
@@ -24,6 +26,8 @@ let Benchmarks = ../../Constants/Benchmarks.dhall
 
 let SelectFiles = ../../Lib/SelectFiles.dhall
 
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
+
 let Spec =
       { Type =
           { key : Text
@@ -67,6 +71,7 @@ let command
                 "Perf: ${spec.label} ${PipelineMode.capitalName spec.mode}"
             , key = spec.key
             , target = spec.size
+            , soft_fail = Some (B/SoftFail.Boolean True)
             , docker = None Docker.Type
             , depends_on = spec.dependsOn
             }

buildkite/src/Constants/DebianVersions.dhall

@@ -75,6 +75,7 @@ let minimalDirtyWhen =
       , S.exactly "buildkite/src/Command/HardforkPackageGeneration" "dhall"
       , S.exactly "buildkite/src/Command/MinaArtifact" "dhall"
       , S.exactly "buildkite/src/Command/PatchArchiveTest" "dhall"
+      , S.exactly "buildkite/src/Command/Bench/Base" "dhall"
       , S.exactly "buildkite/src/Command/ReplayerTest" "dhall"
       , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact")
       , S.strictlyStart (S.contains "dockerfiles/stages")

buildkite/src/README.md

@@ -1,6 +1,6 @@
 # Buildkite CI
 
-This folder contains all dhall code which is an backbone for our CI related code for buildkite.
+This folder contains all dhall code which is a backbone for our CI related code for buildkite.
 
 # Structure
 
@@ -53,7 +53,7 @@ User defined value which describe current pipeline chunk of jobs to be executed.
 - coverage gathering - which gathers coverage artifacts and uploads it to coveralls.io
 
 To reach above pipeline configuration below configuration can be provided:
-(non important attributes where omitted)
+(non-important attributes were omitted)
 ```
 steps:
   - commands:
@@ -204,4 +204,4 @@ We want only to move dockers from gcr to dockerhub without changing version. Cur
   - "NEW_VERSION=3.0.0-dc6bf78"
   - "CODENAMES=Focal,Bullseye"
   - "PUBLISH=1"
-```
+```

docs/test_dsl_spec.md

@@ -192,7 +192,7 @@ DSL.List.iter (left_partition @ right_partition) ~f:destroy
 
 ALTERNATIVE TEST: keep two partitions separate with a fixed topology, with 1-2 intermediate
 nodes bridging the networks, then take the other bridge offline temporarily and then have them
-rejoin the network without topologoical restrictions and see if the chains reconverge
+rejoin the network without topological restrictions and see if the chains reconverge
 
 ##### Basic Hard Fork Test
 

docs/testnet-guardian-runbook.md

@@ -144,7 +144,7 @@ Following is the list of events that could occur after the release
 |--|------|--------|-------------|--|
 | 1| Issues preventing the users from completing the task | Since, we already tested all the challenges, either the user did the task incorrectly or in a different way that was missed by the engineering team. Usually, the community responds quickly to issues involving the challenges. Let the user know of the alternative way to finish the task. If the errors are real protocol/product bugs, create an issue and request the user to attach coda logs to the issue| Minor | |
 | 2| Users' nodes crashing intermittently | If these are not one of the known bugs then create an issue for the same. Request the user to attach the latest crash report| Minor | |
-| 3| Users' nodes crashing persistently | If it is for a specific user, might be that they did something differently when starting the node or their environment is not as expected. For example, connection timeouts (eventually causing the dameons to crash) between daemon and prover or daemon and snark workers could be because of resource constraints. If the cause is not determined, create an issue and request the user to attach the crash report | Major | Engineering team |
+| 3| Users' nodes crashing persistently | If it is for a specific user, might be that they did something differently when starting the node or their environment is not as expected. For example, connection timeouts (eventually causing the daemons to crash) between daemon and prover or daemon and snark workers could be because of resource constraints. If the cause is not determined, create an issue and request the user to attach the crash report | Major | Engineering team |
 |4| Unstable testnet | Create an issue for the protocol team to investigate. Coordinate with the owners of this event to discuss further actions based on the findings by the protocol team   | Critical | Aneesha, Brandon, Engineer investigating the issue|
 
 ## Change Log

nix/README.md

@@ -605,7 +605,7 @@ happen when you try to build anything inside the pure shell. It happens because
 the stack size for every process is limited, and it is shared between the
 current environment, the argument list, and some other things. Therefore, if
 your environment takes up too much space, not enough is left for the arguments.
-The way to fix the error is to unset some of the bigger enviornment variables,
+The way to fix the error is to unset some of the bigger environment variables,
 perhaps with
 
 ```bash

nix/ocaml.nix

@@ -254,7 +254,7 @@ let
         done
       '') package.outputs);
 
-      # Derivation which has all Mina's dependencies in it, and creates an empty output if the command succeds.
+      # Derivation which has all Mina's dependencies in it, and creates an empty output if the command succeeds.
       # Useful for unit tests.
       runMinaCheck = { name ? "check", extraInputs ? [ ], extraArgs ? { }, }:
         check:

nix/rust.nix

@@ -5,7 +5,7 @@ let
     prev.makeRustPlatform {
       cargo = rust;
       rustc = rust;
-      # override stdenv.targetPlatform here, if neccesary
+      # override stdenv.targetPlatform here, if necessary
     };
   toolchainHashes = {
     "1.72" = "sha256-dxE7lmCFWlq0nl/wKcmYvpP9zqQbBitAQgZ1zx9Ooik=";

rfcs/0016-transition-frontier-persistence.md

@@ -8,7 +8,7 @@ This RFC proposes a new system for persisting the transition frontier's state to
 ## Motivation
 [motivation]: #motivation
 
-The Transition Frontier is too large of a data structure to just blindly serialize and write to disk. Under non optimal network scenarios, we expect the upper bound of the data structure to be >100Gb. Even if the structure were smaller, we cannot write the structure out to disk every time we mutate it as the speed of the transition frontier data structure is critical to the systems ability to prevent DDoS attacks. Therefore, a more robust and effecient system is required to persist the Transition Frontier to disk without negatively effecting the speed of operations on the in memory copy of the Transition Frontier.
+The Transition Frontier is too large of a data structure to just blindly serialize and write to disk. Under non-optimal network scenarios, we expect the upper bound of the data structure to be >100Gb. Even if the structure were smaller, we cannot write the structure out to disk every time we mutate it as the speed of the transition frontier data structure is critical to the systems ability to prevent DDoS attacks. Therefore, a more robust and effecient system is required to persist the Transition Frontier to disk without negatively effecting the speed of operations on the in memory copy of the Transition Frontier.
 
 ## Detailed design
 [detailed-design]: #detailed-design
@@ -30,7 +30,7 @@ As actions are performed on the Transition Frontier, diffs are emitted and store
 
 Having two different mechanisms for writing the same data can be tricky as there can be bugs in one of the two mechanisms that would cause the data structures to become desynchronized. In order to help prevent this, we can introduce an incremental hash on top of the Transition Frontier which can be updated upon each diff application. This hash will give a direct and easy way to compare the structural equality of the two data structures. Being incremental, however, also means that the order of diff application needs to be the same across both data structures, so care needs to be taken with that ordering. Therefore, in a sense, this hash will represent the structure and content of the data structure, as well as the order in which actions were taken to get there. We only care about the former in our case, and the latter is just a consequence of the hash being incremental.
 
-In order to calculate this hash correctly, we need to introduce a new concept to a diff, which is that of a diff mutant. Each diff represents some mutation to perform on the Transition Frontier, however not every diff will contain the enough information by itself to encapsulate the state of the data structure after the mutation occurs. For example, setting a balance on an account in two implementations of the data structure does not guarantee that the accounts in each a equal as there are other fields on the account besides that. This is where the concept of a diff mutant comes in. The mutant of a diff is the set of all modified values in the data structure after the diff has been applied. Using this, we can create a proper incremental diff which will truly ensure our data structures are in sync.
+In order to calculate this hash correctly, we need to introduce a new concept to a diff, which is that of a diff mutant. Each diff represents some mutation to perform on the Transition Frontier, however not every diff will contain the enough information by itself to encapsulate the state of the data structure after the mutation occurs. For example, setting a balance on an account in two implementations of the data structure does not guarantee that the accounts in each an equal as there are other fields on the account besides that. This is where the concept of a diff mutant comes in. The mutant of a diff is the set of all modified values in the data structure after the diff has been applied. Using this, we can create a proper incremental diff which will truly ensure our data structures are in sync.
 
 These hashes will be Sha256 as there is no reason to use the Pedersen hashing algorithm we use in the rest of our code since none of this information needs to be snarked. The formula for calculating a new hash `h'` given an old hash `h` and a diff `diff` is as follows: `h' = sha256 h diff (Diff.mutant diff)`.
 

rfcs/0020-transition-frontier-extensions-2.md

@@ -28,7 +28,7 @@ See [#1585](https://github.com/CodaProtocol/coda/pull/1585) for early discussion
 
 ### Extensions Redefined
 
-A Transition Frontier Extension is an stateful, incremental view on the state of a Transiton Frontier. When a Transition Frontier is initialized, all of its extensions are also initialized using the Transition Frontier's root. Every mutation performed is represented as a list of diffs, and when the Transition Frontier updates, each Extension is notified of this list of diffs synchronously. Transition Frontier Extensions will notify the Transition Frontier if there was a update to the Extension's view when handling the diffs. If an Extension's view is updated, then a synchronous event is broadcast internally with the new view of that Extension. A Transition Frontier Extension has access to the Transition Frontier so that it can query and calculate information it requires when it handles diffs.
+A Transition Frontier Extension is a stateful, incremental view on the state of a Transiton Frontier. When a Transition Frontier is initialized, all of its extensions are also initialized using the Transition Frontier's root. Every mutation performed is represented as a list of diffs, and when the Transition Frontier updates, each Extension is notified of this list of diffs synchronously. Transition Frontier Extensions will notify the Transition Frontier if there was a update to the Extension's view when handling the diffs. If an Extension's view is updated, then a synchronous event is broadcast internally with the new view of that Extension. A Transition Frontier Extension has access to the Transition Frontier so that it can query and calculate information it requires when it handles diffs.
 
 ### Extension Guidelines
 

rfcs/0026-transition-caching.md

@@ -8,7 +8,7 @@ A new transition caching logic for the transition router which aims to track tra
 ## Motivation
 [motivation]: #motivation
 
-Within the transition router system, the only check for duplicate transitions is performed by the transition validator, and each transition is only checked against the transitions which are currently in the transition frontier. However, there are two types of duplicate transitions which are not being checked for: transitions which are still being processed by the system (either in the processor pipe or in the catchup scheduler and catchup thread), and transitions which have been determined to be invalid. In the case of the former, the system ends up processing more transitions than necessary, and the number of duplicated processing increases along with the networks size. In the case of the latter, the system is opened up for DDoS attacks since an adversary could continously send transitions with valid proofs but invalid staged ledger diffs, causing each node to spend a significant enough amount of time before invalidating the transition each time it recieves it.
+Within the transition router system, the only check for duplicate transitions is performed by the transition validator, and each transition is only checked against the transitions which are currently in the transition frontier. However, there are two types of duplicate transitions which are not being checked for: transitions which are still being processed by the system (either in the processor pipe or in the catchup scheduler and catchup thread), and transitions which have been determined to be invalid. In the case of the former, the system ends up processing more transitions than necessary, and the number of duplicated processing increases along with the networks size. In the case of the latter, the system is opened up for DDoS attacks since an adversary could continuously send transitions with valid proofs but invalid staged ledger diffs, causing each node to spend a significant enough amount of time before invalidating the transition each time it recieves it.
 
 NOTE: This RFC has been re-scoped to only address duplicate transitions already being processed and not transitions which were previously determined to be invalid.
 

src/app/dump_blocks/encoding.ml

@@ -50,9 +50,10 @@ module Sexp_block : S with type t = Mina_block.t = struct
 
   let of_breadcrumb = block_of_breadcrumb
 
-  let to_string b = Mina_block.sexp_of_t b |> Sexp.to_string |> append_newline
+  let to_string b =
+    Mina_block.Stable.Latest.sexp_of_t b |> Sexp.to_string |> append_newline
 
-  let of_string s = Sexp.of_string s |> Mina_block.t_of_sexp
+  let of_string s = Sexp.of_string s |> Mina_block.Stable.Latest.t_of_sexp
 end
 
 module Binary_block : S with type t = Mina_block.t = struct

src/lib/block_producer/block_producer.ml

@@ -420,26 +420,6 @@ let generate_next_state ~commit_id ~zkapp_cmd_limit ~constraint_constants
               in
               Some (protocol_state, internal_transition, witness) ) )
 
-module Precomputed = struct
-  type t = Precomputed.t =
-    { scheduled_time : Block_time.t
-    ; protocol_state : Protocol_state.value
-    ; protocol_state_proof : Proof.t
-    ; staged_ledger_diff : Staged_ledger_diff.t
-    ; delta_transition_chain_proof :
-        Frozen_ledger_hash.t * Frozen_ledger_hash.t list
-    ; protocol_version : Protocol_version.t
-    ; proposed_protocol_version : Protocol_version.t option
-    ; accounts_accessed : (int * Account.t) list
-    ; accounts_created : (Account_id.t * Currency.Fee.t) list
-    ; tokens_used : (Token_id.t * Account_id.t option) list
-    }
-
-  let sexp_of_t = Precomputed.sexp_of_t
-
-  let t_of_sexp = Precomputed.t_of_sexp
-end
-
 let handle_block_production_errors ~logger ~rejected_blocks_logger
     ~time_taken:span ~previous_protocol_state ~protocol_state x =
   let transition_error_msg_prefix = "Validation failed: " in

src/lib/disk_cache/disk_cache.mli

@@ -1,20 +1 @@
-open Core_kernel
-open Async
-
-module Make : functor (T : Binable.S) -> sig
-  type t
-
-  (** Initialize the on-disk cache explicitly before interactions with it take place. *)
-  val initialize :
-       string
-    -> logger:Logger.t
-    -> (t, [> `Initialization_error of Error.t ]) Deferred.Result.t
-
-  type id
-
-  (** Put the value to disk, return an identifier that is associated with a special handler in GC. *)
-  val put : t -> T.t -> id
-
-  (** Read from the cache, crashing if the value cannot be found. *)
-  val get : t -> id -> T.t
-end
+module Make : Disk_cache_intf.F

src/lib/disk_cache/dune

@@ -1,14 +1,16 @@
 (library
  (public_name disk_cache)
  (virtual_modules disk_cache)
- (default_implementation disk_cache.filesystem)
+ (default_implementation disk_cache.identity)
  (libraries
    ;; opam libraries
-   core_kernel
+   core
    async
    ;; local libraries
+   file_system
    logger
+   disk_cache.intf
    )
  (preprocess
-  (pps ppx_mina ppx_version))
+  (pps ppx_mina ppx_version ppx_jane))
  (instrumentation (backend bisect_ppx)))