Merge remote-tracking branch 'origin/trunk' into series/M3

.gitattributes

@@ -0,0 +1,2 @@
+* text=auto
+*.uc binary

.github/workflows/ci.yaml

@@ -16,16 +16,23 @@ jobs:
   build:
     name: ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    defaults: 
+      run:
+        working-directory: unison
     strategy:
       # Run each build to completion, regardless of if any have failed
       fail-fast: false
       matrix:
         os:
+          # temporarily disable non-windows builds to speed up iteration time
           - ubuntu-20.04
           - macOS-11.0
           - macOS-10.15
+          - windows-2019
     steps:
       - uses: actions/checkout@v2
+        with:
+          path: unison
 
       # The number towards the beginning of the cache keys allow you to manually avoid using a previous cache.
       # GitHub will automatically delete caches that haven't been accessed in 7 days, but there is no way to
@@ -67,14 +74,18 @@ jobs:
       # so this is split into two steps, only one of which will run on any particular build.
       - name: install stack (Linux)
         if: runner.os == 'Linux'
+        working-directory: ${{ github.workspace }}
         run: |
+          mkdir stack && cd stack
           curl -L https://github.com/commercialhaskell/stack/releases/download/v2.5.1/stack-2.5.1-linux-x86_64.tar.gz | tar -xz
-          echo "$HOME/stack-2.5.1-linux-x86_64/" >> $GITHUB_PATH
+          echo "$PWD/stack-2.5.1-linux-x86_64/" >> $GITHUB_PATH
       - name: install stack (macOS)
+        working-directory: ${{ github.workspace }}
         if: runner.os == 'macOS'
         run: |
+          mkdir stack && cd stack
           curl -L https://github.com/commercialhaskell/stack/releases/download/v2.5.1/stack-2.5.1-osx-x86_64.tar.gz | tar -xz
-          echo "$HOME/stack-2.5.1-osx-x86_64/" >> $GITHUB_PATH
+          echo "$PWD/stack-2.5.1-osx-x86_64/" >> $GITHUB_PATH
 
       # One of the transcripts fails if the user's git name hasn't been set.
       - name: set git user info
@@ -103,8 +114,10 @@ jobs:
       - name: transcripts
         run: |
           stack --no-terminal exec transcripts
-          git diff
-          x=`git status --porcelain -uno` bash -c 'if [[ -n $x ]]; then echo "$x" && false; fi'
+          # Add all changes to the index for when we diff.
+          git add --all
+          # Fail if any transcripts cause git diffs.
+          git diff --cached --ignore-cr-at-eol --exit-code
       - name: prettyprint-round-trip
         run: stack --no-terminal exec unison transcript unison-src/transcripts-round-trip/main.md
       - name: integration-tests

.github/workflows/release.yaml

@@ -12,6 +12,7 @@ jobs:
     needs:
       - build_linux
       - build_macos
+      - build_windows
 
     steps:
       - uses: actions/checkout@v2
@@ -48,12 +49,6 @@ jobs:
           curl -L https://github.com/commercialhaskell/stack/releases/download/v2.5.1/stack-2.5.1-linux-x86_64.tar.gz | tar -xz
           echo "$HOME/stack-2.5.1-linux-x86_64/" >> $GITHUB_PATH
 
-      # One of the transcripts fails if the user's git name hasn't been set.
-      - name: set git user info
-        run: |
-          git config --global user.name "GitHub Actions"
-          git config --global user.email "[email protected]"
-
       - name: build
         run: stack --no-terminal build --flag unison-parser-typechecker:optimized
 
@@ -84,12 +79,6 @@ jobs:
           curl -L https://github.com/commercialhaskell/stack/releases/download/v2.5.1/stack-2.5.1-osx-x86_64.tar.gz | tar -xz
           echo "$HOME/stack-2.5.1-osx-x86_64/" >> $GITHUB_PATH
 
-      # One of the transcripts fails if the user's git name hasn't been set.
-      - name: set git user info
-        run: |
-          git config --global user.name "GitHub Actions"
-          git config --global user.email "[email protected]"
-
       - name: remove ~/.stack/setup-exe-cache on macOS
         run: rm -rf ~/.stack/setup-exe-cache
 
@@ -111,3 +100,33 @@ jobs:
           if-no-files-found: error
           name: build-macos
           path: ucm-macos.tar.gz
+
+  build_windows:
+    name: "build_windows"
+    runs-on: windows-2019
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: install stack
+        run: |
+          curl -L https://github.com/commercialhaskell/stack/releases/download/v2.5.1/stack-2.5.1-osx-x86_64.tar.gz | tar -xz
+          echo "$HOME/stack-2.5.1-osx-x86_64/" >> $GITHUB_PATH
+
+      - name: build
+        run: stack --no-terminal build --flag unison-parser-typechecker:optimized
+
+      - name: fetch latest codebase-ui and package with ucm
+        run: |
+          mkdir -p /tmp/ucm/ui
+          UCM=$(stack path | awk '/local-install-root/{print $2}')/bin/unison
+          cp $UCM /tmp/ucm/ucm
+          wget -O/tmp/unisonLocal.zip https://github.com/unisonweb/codebase-ui/releases/download/latest/unisonLocal.zip
+          unzip -d /tmp/ucm/ui /tmp/unisonLocal.zip
+          tar -c -z -f ucm-windows.tar.gz -C /tmp/ucm .
+
+      - name: Upload windows artifact
+        uses: actions/upload-artifact@v2
+        with:
+          if-no-files-found: error
+          name: build-windows
+          path: ucm-windows.tar.gz

.mergify.yml

@@ -5,6 +5,7 @@ pull_request_rules:
       # - status-success=ubuntu-18.04
       - status-success=macOS-11.0
       - status-success=macOS-10.15
+      - status-success=windows-2019
       - label=ready-to-merge
       - "#approved-reviews-by>=1"
     actions:

CONTRIBUTORS.markdown

@@ -66,3 +66,4 @@ The format for this list: name, GitHub handle, and then optional blurb about wha
 * Sameer Kolhar (@kolharsam)
 * Nicole Prindle (@nprindle)
 * Harald Gliebe (@hagl)
+* Phil de Joux (@philderbeast)

codebase2/codebase-sqlite/U/Codebase/Sqlite/Branch/Diff.hs

@@ -1,38 +1,45 @@
-{- ORMOLU_DISABLE -} -- Remove this when the file is ready to be auto-formatted
-{-# LANGUAGE RankNTypes #-}
-{-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE DeriveFunctor #-}
 {-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE ViewPatterns #-}
 
 module U.Codebase.Sqlite.Branch.Diff where
 
+import Data.Bifunctor (Bifunctor (bimap))
 import qualified Data.List as List
 import Data.Map (Map)
 import qualified Data.Map as Map
 import Data.Set (Set)
+import qualified Data.Set as Set
 import U.Codebase.Reference (Reference')
 import U.Codebase.Referent (Referent')
 import U.Codebase.Sqlite.DbId (BranchObjectId, CausalHashId, ObjectId, PatchObjectId, TextId)
 import U.Codebase.Sqlite.LocalIds (LocalBranchChildId, LocalDefnId, LocalPatchObjectId, LocalTextId)
 import qualified Unison.Util.Map as Map
-import Data.Bifunctor (Bifunctor(bimap))
-import qualified Data.Set as Set
 
 type LocalDiff = Diff' LocalTextId LocalDefnId LocalPatchObjectId LocalBranchChildId
+
 type Diff = Diff' TextId ObjectId PatchObjectId (BranchObjectId, CausalHashId)
 
-data DefinitionOp' r = RemoveDef | AddDefWithMetadata (Set r) | AlterDefMetadata (AddRemove r) deriving Show
+data DefinitionOp' r = RemoveDef | AddDefWithMetadata (Set r) | AlterDefMetadata (AddRemove r) deriving (Show)
+
 data PatchOp' p = PatchRemove | PatchAddReplace p deriving (Functor, Show)
+
 data ChildOp' c = ChildRemove | ChildAddReplace c deriving (Functor, Show)
+
 type AddRemove a = Map a Bool
 
 type LocalDefinitionOp = DefinitionOp' (Metadata LocalTextId LocalDefnId)
+
 type LocalPatchOp = PatchOp' LocalPatchObjectId
+
 type LocalChildOp = ChildOp' LocalBranchChildId
 
 type DefinitionOp = DefinitionOp' (Metadata TextId ObjectId)
+
 type PatchOp = PatchOp' PatchObjectId
+
 type ChildOp = ChildOp' (BranchObjectId, CausalHashId)
 
 addsRemoves :: AddRemove a -> ([a], [a])
@@ -48,7 +55,7 @@ data Diff' t h p c = Diff
     patches :: Map t (PatchOp' p),
     children :: Map t (ChildOp' c)
   }
-  deriving Show
+  deriving (Show)
 
 type Metadata t h = Reference' t h
 

codebase2/codebase-sqlite/U/Codebase/Sqlite/Branch/Format.hs

@@ -1,26 +1,68 @@
-{- ORMOLU_DISABLE -} -- Remove this when the file is ready to be auto-formatted
-module U.Codebase.Sqlite.Branch.Format where
+module U.Codebase.Sqlite.Branch.Format
+  ( BranchFormat (..),
+    BranchLocalIds (..),
+    SyncBranchFormat (..),
+    localToDbBranch,
+    localToDbDiff,
+    -- dbToLocalDiff,
+  )
+where
 
 import Data.Vector (Vector)
-import U.Codebase.Sqlite.Branch.Diff (LocalDiff)
-import U.Codebase.Sqlite.Branch.Full (LocalBranch)
-import U.Codebase.Sqlite.DbId (CausalHashId, BranchObjectId, ObjectId, PatchObjectId, TextId)
-import Data.ByteString (ByteString)
+import qualified Data.Vector as Vector
+import U.Codebase.Sqlite.Branch.Diff (Diff, LocalDiff)
+import qualified U.Codebase.Sqlite.Branch.Diff as Branch.Diff
+import U.Codebase.Sqlite.Branch.Full (DbBranch, LocalBranch)
+import qualified U.Codebase.Sqlite.Branch.Full as Branch.Full
+import U.Codebase.Sqlite.DbId (BranchObjectId, CausalHashId, ObjectId, PatchObjectId, TextId)
+import U.Codebase.Sqlite.LocalIds
+  ( LocalBranchChildId (..),
+    LocalDefnId (..),
+    LocalPatchObjectId (..),
+    LocalTextId (..),
+  )
+import Unison.Prelude
 
--- |you can use the exact same `BranchLocalIds` when converting between `Full` and `Diff`
+-- | A 'BranchFormat' is a deserialized namespace object (@object.bytes@).
+--
+-- you can use the exact same `BranchLocalIds` when converting between `Full` and `Diff`
 data BranchFormat
   = Full BranchLocalIds LocalBranch
   | Diff BranchObjectId BranchLocalIds LocalDiff
-  deriving Show
+  deriving (Show)
 
+-- | A 'BranchLocalIds' is a mapping between local ids (local to this object) encoded as offsets, and actual database ids.
+--
+-- For example, a @branchTextLookup@ vector of @[50, 74]@ means "local id 0 corresponds to database text id 50, and
+-- local id 1 corresponds to database text id 74".
 data BranchLocalIds = LocalIds
   { branchTextLookup :: Vector TextId,
     branchDefnLookup :: Vector ObjectId,
     branchPatchLookup :: Vector PatchObjectId,
     branchChildLookup :: Vector (BranchObjectId, CausalHashId)
   }
-  deriving Show
+  deriving (Show)
 
 data SyncBranchFormat
   = SyncFull BranchLocalIds ByteString
   | SyncDiff BranchObjectId BranchLocalIds ByteString
+
+localToDbBranch :: BranchLocalIds -> LocalBranch -> DbBranch
+localToDbBranch li =
+  Branch.Full.quadmap (lookupBranchLocalText li) (lookupBranchLocalDefn li) (lookupBranchLocalPatch li) (lookupBranchLocalChild li)
+
+localToDbDiff :: BranchLocalIds -> LocalDiff -> Diff
+localToDbDiff li =
+  Branch.Diff.quadmap (lookupBranchLocalText li) (lookupBranchLocalDefn li) (lookupBranchLocalPatch li) (lookupBranchLocalChild li)
+
+lookupBranchLocalText :: BranchLocalIds -> LocalTextId -> TextId
+lookupBranchLocalText li (LocalTextId w) = branchTextLookup li Vector.! fromIntegral w
+
+lookupBranchLocalDefn :: BranchLocalIds -> LocalDefnId -> ObjectId
+lookupBranchLocalDefn li (LocalDefnId w) = branchDefnLookup li Vector.! fromIntegral w
+
+lookupBranchLocalPatch :: BranchLocalIds -> LocalPatchObjectId -> PatchObjectId
+lookupBranchLocalPatch li (LocalPatchObjectId w) = branchPatchLookup li Vector.! fromIntegral w
+
+lookupBranchLocalChild :: BranchLocalIds -> LocalBranchChildId -> (BranchObjectId, CausalHashId)
+lookupBranchLocalChild li (LocalBranchChildId w) = branchChildLookup li Vector.! fromIntegral w

codebase2/codebase-sqlite/U/Codebase/Sqlite/Branch/Full.hs

@@ -1,21 +1,44 @@
-{- ORMOLU_DISABLE -} -- Remove this when the file is ready to be auto-formatted
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DeriveGeneric #-}
 {-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeApplications #-}
 
 module U.Codebase.Sqlite.Branch.Full where
 
-import Data.Map (Map)
-import Data.Set (Set)
+import Control.Lens
+import qualified Data.Map as Map
+import qualified Data.Set as Set
 import U.Codebase.Reference (Reference')
+import qualified U.Codebase.Reference as Reference
 import U.Codebase.Referent (Referent')
 import U.Codebase.Sqlite.DbId (BranchObjectId, CausalHashId, ObjectId, PatchObjectId, TextId)
 import U.Codebase.Sqlite.LocalIds (LocalBranchChildId, LocalDefnId, LocalPatchObjectId, LocalTextId)
+import Unison.Prelude
 import qualified Unison.Util.Map as Map
-import Data.Bifunctor (Bifunctor(bimap))
-import qualified Data.Set as Set
+import qualified Unison.Util.Set as Set
 
+-- |
+-- @
+-- Branch
+--   { terms :: Map LocalTextId (Map LocalReferent LocalMetadataSet),
+--     types :: Map LocalTextId (Map LocalReference LocalMetadataSet),
+--     patches :: Map LocalTextId LocalPatchObjectId,
+--     children :: Map LocalTextId LocalBranchChildId
+--   }
+-- @
 type LocalBranch = Branch' LocalTextId LocalDefnId LocalPatchObjectId LocalBranchChildId
 
+-- |
+-- @
+-- Branch
+--   { terms :: Map TextId (Map Referent DbMetadataSet),
+--     types :: Map TextId (Map Reference DbMetadataSet),
+--     patches :: Map TextId PatchObjectId,
+--     children :: Map TextId (BranchObjectId, CausalHashId)
+--   }
+-- @
 type DbBranch = Branch' TextId ObjectId PatchObjectId (BranchObjectId, CausalHashId)
 
 type Referent'' t h = Referent' (Reference' t h) (Reference' t h)
@@ -26,14 +49,38 @@ data Branch' t h p c = Branch
     patches :: Map t p,
     children :: Map t c
   }
-  deriving Show
+  deriving (Show, Generic)
+
+emptyBranch :: Branch' t h p c
+emptyBranch = Branch Map.empty Map.empty Map.empty Map.empty
+
+branchHashes_ :: (Ord h', Ord t, Ord h) => Traversal (Branch' t h p c) (Branch' t h' p c) h h'
+branchHashes_ f Branch {..} = do
+  newTerms <- for terms (Map.bitraversed both metadataSetFormatReferences_ . Reference.h_ %%~ f)
+  newTypes <- for types (Map.bitraversed id metadataSetFormatReferences_ . Reference.h_ %%~ f)
+  pure Branch {terms = newTerms, types = newTypes, patches, children}
+
+patches_ :: Traversal (Branch' t h p c) (Branch' t h p' c) p p'
+patches_ f Branch {..} = (\newPatches -> Branch terms types newPatches children) <$> traverse f patches
+
+childrenHashes_ :: Traversal (Branch' t h p c) (Branch' t h p c') c c'
+childrenHashes_ f Branch {..} = Branch terms types patches <$> traverse f children
+
+branchCausalHashes_ :: Traversal (Branch' t h p c) (Branch' t h p c') c c'
+branchCausalHashes_ f Branch {..} =
+  Branch terms types patches <$> traverse f children
 
 type LocalMetadataSet = MetadataSetFormat' LocalTextId LocalDefnId
 
 type DbMetadataSet = MetadataSetFormat' TextId ObjectId
 
 data MetadataSetFormat' t h = Inline (Set (Reference' t h))
-  deriving Show
+  deriving (Show)
+
+metadataSetFormatReferences_ ::
+  (Ord t, Ord h, Ord h') =>
+  Traversal (MetadataSetFormat' t h) (MetadataSetFormat' t h') (Reference' t h) (Reference' t h')
+metadataSetFormatReferences_ f (Inline refs) = Inline <$> Set.traverse f refs
 
 quadmap :: forall t h p c t' h' p' c'. (Ord t', Ord h') => (t -> t') -> (h -> h') -> (p -> p') -> (c -> c') -> Branch' t h p c -> Branch' t' h' p' c'
 quadmap ft fh fp fc (Branch terms types patches children) =