Merge pull request darklang#5545 from StachuDotNet/scm

refactors: package manager, sync, and dev UX across CLI + VS Code

Author: StachuDotNet

Dockerfile

@@ -320,7 +320,7 @@ ENV NUGET_SCRATCH=/tmp/NuGetScratch
 # TODO Occasionally, check https://ziglang.org/download to see if we're using the latest version
 ENV ZIG_VERSION=0.11.0
 ENV ZIG_ARM64_MINISIG="RUSGOq2NVecA2XPwbgbN5SvU46UcCmhhfcfrjVC+YvcwUcjAYfIXQmqE//df1Mes7iyGZvGoy2+PSJ8pog7QGLE+3nvP8gtlSAs="
-ENV ZIG=AMD64_MINISIG="RUSGOq2NVecA2X2did6P61CXthPLZEUwi07GDWQ2MWU58W+asm3v85+PRVHN5SljhdsKoAMmbg4fdyseAcbVZayGaV1Iv6chcgE="
+ENV ZIG_AMD64_MINISIG="RUSGOq2NVecA2X2did6P61CXthPLZEUwi07GDWQ2MWU58W+asm3v85+PRVHN5SljhdsKoAMmbg4fdyseAcbVZayGaV1Iv6chcgE="
 #############
 RUN set -e; \
   case ${TARGETARCH} in \

backend/migrations/20251015_192755_package_schema_rewrite.sql

@@ -0,0 +1,115 @@
+-- Rewrites package storage for branch support and sync tracking
+-- Old: 3 tables (package_types_v0, package_values_v0, package_fns_v0)
+--      combining id, name/location, PT and RT
+-- New: Separate content storage from naming:
+--      - Content tables: package_types/values/functions (content-addressed)
+--      - Naming table: locations (maps owner.modules.name to item_id)
+--      - Branches: track work-in-progress
+--      - Package ops: source of truth for all changes
+
+-- Drop old package tables
+DROP TABLE IF EXISTS package_types_v0;
+DROP TABLE IF EXISTS package_values_v0;
+DROP TABLE IF EXISTS package_functions_v0;
+
+
+-- Content tables store definitions (content-addressed, no naming)
+CREATE TABLE IF NOT EXISTS package_types (
+  id TEXT PRIMARY KEY,
+  pt_def BLOB NOT NULL,
+  rt_def BLOB NOT NULL,
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE TABLE IF NOT EXISTS package_values (
+  id TEXT PRIMARY KEY,
+  pt_def BLOB NOT NULL,
+  rt_dval BLOB NOT NULL,
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE TABLE IF NOT EXISTS package_functions (
+  id TEXT PRIMARY KEY,
+  pt_def BLOB NOT NULL,
+  rt_instrs BLOB NOT NULL,
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Locations table: name resolution
+-- Maps (owner, modules, name, branch_id) -> item_id
+-- Supports branch-specific overrides and deprecation
+CREATE TABLE IF NOT EXISTS locations (
+  location_id TEXT PRIMARY KEY,
+  item_id TEXT NOT NULL, -- refs package_types/values/functions
+  branch_id TEXT NULL, -- NULL = main, non-null = branch override
+  owner TEXT NOT NULL, -- e.g., "Darklang"
+  modules TEXT NOT NULL, -- e.g., "Stdlib.List"
+  name TEXT NOT NULL, -- e.g., "map"
+  item_type TEXT NOT NULL, -- 'fn', 'type', or 'value'
+  created_at TIMESTAMP NOT NULL DEFAULT (datetime('now')),
+  deprecated_at TIMESTAMP NULL -- NULL = active
+  -- No unique constraint: branches can have conflicting names
+);
+
+-- Indexes for name lookups, module searches, and branch queries
+CREATE INDEX IF NOT EXISTS idx_locations_lookup
+  ON locations(owner, modules, name, branch_id) WHERE deprecated_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_locations_module
+  ON locations(owner, modules) WHERE deprecated_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_locations_conflicts
+  ON locations(owner, modules, name, branch_id)
+  WHERE deprecated_at IS NULL AND branch_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_locations_shadowing
+  ON locations(owner, modules, name, item_type, branch_id, deprecated_at);
+CREATE INDEX IF NOT EXISTS idx_locations_branch
+  ON locations(branch_id) WHERE branch_id IS NOT NULL;
+
+
+CREATE TABLE IF NOT EXISTS branches (
+  id TEXT PRIMARY KEY,
+  name TEXT NOT NULL,
+  created_at TIMESTAMP NOT NULL,
+  --CLEANUP consider an `updated_at` col
+  merged_at TIMESTAMP NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_branches_name ON branches(name);
+
+-- Package ops: the source of truth for all package changes
+-- Other tables (types/values/functions/locations) are projections
+CREATE TABLE IF NOT EXISTS package_ops (
+  id TEXT PRIMARY KEY,
+  branch_id TEXT NULL, -- NULL = merged to main
+  op_blob BLOB NOT NULL, -- Serialized PackageOp
+  applied BOOLEAN NOT NULL DEFAULT FALSE, -- Track whether op has been processed
+  created_at TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_package_ops_branch ON package_ops(branch_id);
+CREATE INDEX IF NOT EXISTS idx_package_ops_created ON package_ops(created_at);
+CREATE INDEX IF NOT EXISTS idx_package_ops_applied ON package_ops(applied) WHERE applied = FALSE;
+
+-- Instances table tracks remote Darklang instances for syncing
+CREATE TABLE IF NOT EXISTS instances (
+  id TEXT PRIMARY KEY,
+  name TEXT NOT NULL UNIQUE,
+  url TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_instances_name ON instances(name);
+
+-- Syncs table tracks synchronization history with remote instances
+CREATE TABLE IF NOT EXISTS syncs (
+  id TEXT PRIMARY KEY,                    -- UUID
+  instance_id TEXT NOT NULL,              -- UUID reference to instances table
+  synced_at TIMESTAMP NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
+  ops_pushed INTEGER NOT NULL DEFAULT 0,  -- Count of ops sent to remote instance
+  ops_fetched INTEGER NOT NULL DEFAULT 0, -- Count of ops received from remote instance
+  FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_syncs_instance ON syncs(instance_id, synced_at DESC);
+CREATE INDEX IF NOT EXISTS idx_syncs_recent ON syncs(synced_at DESC);
+
+-- Index for package tree queries (speeds up owner.modules LIKE queries)
+CREATE INDEX IF NOT EXISTS idx_locations_owner_modules ON locations(owner, modules);

backend/serialization/vanilla_LibExecution-ProgramTypes-PackageFn-PackageFn_function.json

@@ -1,11 +1,7 @@
 module BuiltinCli.Builtin
 
-open Prelude
-open LibExecution.RuntimeTypes
-
 module Builtin = LibExecution.Builtin
 
-
 let fnRenames : Builtin.FnRenames =
   // old names, new names
   // eg: fn "Http" "respond" 0, fn "Http" "response" 0
@@ -19,5 +15,6 @@ let builtins =
       Libs.Execution.builtins
       Libs.Output.builtins
       Libs.Stdin.builtins
-      Libs.Time.builtins ]
+      Libs.Time.builtins
+      Libs.Terminal.builtins ]
     fnRenames

backend/serialization/vanilla_LibExecution-ProgramTypes-PackageType-PackageType_type.json

@@ -16,11 +16,13 @@
     <Compile Include="Libs/Execution.fs" />
     <Compile Include="Libs/Stdin.fs" />
     <Compile Include="Libs/Time.fs" />
+    <Compile Include="Libs/Terminal.fs" />
     <Compile Include="Builtin.fs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="../Prelude/Prelude.fsproj" />
     <ProjectReference Include="../LibExecution/LibExecution.fsproj" />
+    <ProjectReference Include="../LibConfig/LibConfig.fsproj" />
   </ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
 </Project>

backend/serialization/vanilla_LibExecution-ProgramTypes-PackageValue-PackageValue_value.json

@@ -57,6 +57,20 @@ let fns : List<BuiltInFn> =
         | _ -> incorrectArgs ())
       sqlSpec = NotQueryable
       previewable = Impure
+      deprecated = NotDeprecated }
+
+
+    { name = fn "getBuildHash" 0
+      typeParams = []
+      parameters = [ Param.make "unit" TUnit "" ]
+      returnType = TString
+      description = "Returns the git hash of the current CLI build"
+      fn =
+        function
+        | _, _, [], [ DUnit ] -> uply { return DString LibConfig.Config.buildHash }
+        | _ -> incorrectArgs ()
+      sqlSpec = NotQueryable
+      previewable = Impure
       deprecated = NotDeprecated } ]
 
 

backend/src/BuiltinCli/Builtin.fs

@@ -196,7 +196,13 @@ let fns : List<BuiltInFn> =
             DEnum(typeName, typeName, [], keyCaseName, [])
 
           // Get character representation based on keyboard layout
-          let keyChar = readKey.KeyChar |> string |> DString
+          // Only include keyChar for printable characters, not control characters
+          let keyChar =
+            let ch = readKey.KeyChar
+            if System.Char.IsControl(ch) || ch = '\u0000' then
+              DString "" // Empty string for control/special keys
+            else
+              ch |> string |> DString
 
           let keyRead =
             let typeName =

backend/src/BuiltinCli/BuiltinCli.fsproj

@@ -0,0 +1,139 @@
+module BuiltinCli.Libs.Terminal
+
+open System.Threading.Tasks
+open FSharp.Control.Tasks
+
+open Prelude
+open LibExecution.RuntimeTypes
+open LibExecution.Builtin.Shortcuts
+
+
+let fns : List<BuiltInFn> =
+  [ { name = fn "cliGetTerminalHeight" 0
+      typeParams = []
+      parameters = [ Param.make "unit" TUnit "" ]
+      returnType = TInt64
+      description = "Get the current terminal viewport height in number of lines"
+      fn =
+        (function
+        | _, _, [], [ DUnit ] ->
+          uply {
+            try
+              // First try environment variable (most reliable across different terminals)
+              match System.Environment.GetEnvironmentVariable("LINES") with
+              | null ->
+                // Try Console.WindowHeight
+                let height = System.Console.WindowHeight
+
+                // If we get exactly 24, it might be a default value
+                // Let's try alternative methods
+                if height = 24 then
+                  // On Unix systems, try tput command
+                  if
+                    System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(
+                      System.Runtime.InteropServices.OSPlatform.Linux
+                    )
+                    || System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(
+                      System.Runtime.InteropServices.OSPlatform.OSX
+                    )
+                  then
+                    try
+                      let p = new System.Diagnostics.Process()
+                      p.StartInfo.FileName <- "/bin/sh"
+                      p.StartInfo.Arguments <-
+                        "-c \"tput lines 2>/dev/null || echo 24\""
+                      p.StartInfo.UseShellExecute <- false
+                      p.StartInfo.RedirectStandardOutput <- true
+                      p.StartInfo.CreateNoWindow <- true
+                      if p.Start() then
+                        let output = p.StandardOutput.ReadToEnd().Trim()
+                        p.WaitForExit()
+                        match System.Int32.TryParse(output) with
+                        | true, h when h > 0 -> return DInt64(int64 h)
+                        | _ -> return DInt64(int64 height)
+                      else
+                        return DInt64(int64 height)
+                    with _ ->
+                      return DInt64(int64 height)
+                  else
+                    return DInt64(int64 height)
+                else
+                  return DInt64(int64 height)
+              | lines ->
+                match System.Int32.TryParse(lines) with
+                | true, h when h > 0 -> return DInt64(int64 h)
+                | _ -> return DInt64 24L
+            with _ ->
+              // Fallback if unable to detect terminal size
+              return DInt64 24L
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      deprecated = NotDeprecated }
+
+
+    { name = fn "cliGetTerminalWidth" 0
+      typeParams = []
+      parameters = [ Param.make "unit" TUnit "" ]
+      returnType = TInt64
+      description = "Get the current terminal viewport width in number of columns"
+      fn =
+        (function
+        | _, _, [], [ DUnit ] ->
+          uply {
+            try
+              // First try environment variable (most reliable across different terminals)
+              match System.Environment.GetEnvironmentVariable("COLUMNS") with
+              | null ->
+                // Try Console.WindowWidth
+                let width = System.Console.WindowWidth
+
+                // If we get exactly 80, it might be a default value
+                // Let's try alternative methods
+                if width = 80 then
+                  // On Unix systems, try tput command
+                  if
+                    System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(
+                      System.Runtime.InteropServices.OSPlatform.Linux
+                    )
+                    || System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(
+                      System.Runtime.InteropServices.OSPlatform.OSX
+                    )
+                  then
+                    try
+                      let p = new System.Diagnostics.Process()
+                      p.StartInfo.FileName <- "/bin/sh"
+                      p.StartInfo.Arguments <-
+                        "-c \"tput cols 2>/dev/null || echo 80\""
+                      p.StartInfo.UseShellExecute <- false
+                      p.StartInfo.RedirectStandardOutput <- true
+                      p.StartInfo.CreateNoWindow <- true
+                      if p.Start() then
+                        let output = p.StandardOutput.ReadToEnd().Trim()
+                        p.WaitForExit()
+                        match System.Int32.TryParse(output) with
+                        | true, w when w > 0 -> return DInt64(int64 w)
+                        | _ -> return DInt64(int64 width)
+                      else
+                        return DInt64(int64 width)
+                    with _ ->
+                      return DInt64(int64 width)
+                  else
+                    return DInt64(int64 width)
+                else
+                  return DInt64(int64 width)
+              | columns ->
+                match System.Int32.TryParse(columns) with
+                | true, w when w > 0 -> return DInt64(int64 w)
+                | _ -> return DInt64 80L
+            with _ ->
+              // Fallback if unable to detect terminal size
+              return DInt64 80L
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      deprecated = NotDeprecated } ]
+
+let builtins = LibExecution.Builtin.make [] fns