refactor

src/MinEdLauncher/App.fs

@@ -6,13 +6,13 @@ open System.Runtime.InteropServices
 open System.Security.Cryptography
 open System.Threading
 open MinEdLauncher
+open MinEdLauncher.Http
 open MinEdLauncher.Token
 open FSharp.Control.Tasks.NonAffine
 open System
 open System.Diagnostics
 open System.Threading.Tasks
 open MinEdLauncher.Types
-open MinEdLauncher.HttpClientExtensions
 
 type LoginResult =
 | Success of Api.Connection
@@ -173,117 +173,47 @@ let promptForProductsToUpdate (products: ProductDetails array) =
                 readInput()
     readInput()
 
-let normalizeManifestPartialPath (path: string) =
-    if not (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) then
-        path.Replace('\\', '/')
-    else
-        path
-
-let throttledDownload (semaphore: SemaphoreSlim) (download: 'a -> Task<'b>) input =
+let throttledAction (semaphore: SemaphoreSlim) (action: 'a -> Task<'b>) input =
     input
-    |> Array.map (fun file -> task {
+    |> Array.map (fun item -> task {
         do! semaphore.WaitAsync()
         try
-             return! download(file)
+             return! action(item)
         finally
             semaphore.Release() |> ignore
         })
     |> Task.whenAll
 
-type DownloadProgress = { TotalFiles: int; BytesSoFar: int64; TotalBytes: int64; }
-let downloadFiles (httpClient: HttpClient) (throttler: SemaphoreSlim) destDir (progress: IProgress<DownloadProgress>) cancellationToken (files: Types.ProductManifest.File[]) = task {
-    let combinedTotalBytes = files |> Seq.sumBy (fun f -> int64 f.Size)
-    let combinedBytesSoFar = ref 0L
-    let downloadFile (file: Types.ProductManifest.File) = task {
-        let path = normalizeManifestPartialPath file.Path
-        let dest = Path.Combine(destDir, path)
-        
-        let dirName = Path.GetDirectoryName(dest);
-        if dirName.Length > 0 then
-            Directory.CreateDirectory(dirName) |> ignore
-        
-        let bufferSize = 8192
-        use sha1 = SHA1.Create()
-        use fileStream = new FileStream(dest, FileMode.Create, FileAccess.Write, FileShare.Write, bufferSize, FileOptions.Asynchronous)
-        use cryptoStream = new CryptoStream(fileStream, sha1, CryptoStreamMode.Write) // Calculate hash as file is downloaded
-        let relativeProgress = Progress<int>(fun bytesRead ->
-            let bytesSoFar = Interlocked.Add(combinedBytesSoFar, int64 bytesRead)
-            progress.Report({ TotalFiles = files.Length
-                              BytesSoFar = bytesSoFar
-                              TotalBytes = combinedTotalBytes }))
-        do! httpClient.DownloadAsync(file.Download, cryptoStream, bufferSize, relativeProgress, cancellationToken)
-        cryptoStream.Dispose()
-        let hash = sha1.Hash |> Hex.toString |> String.toLower
-        return dest, hash, file.Hash = hash }
-    
-    try
-        let! result = files |> (throttledDownload throttler downloadFile)
-        return Ok result
-    with e -> return e.ToString() |> Error }
-
 type UpdateProductPaths = { ProductDir: string; ProductCacheDir: string; CacheHashMap: string; ProductHashMap: string }
-let updateProduct (httpClient: HttpClient) (throttler: SemaphoreSlim) cancellationToken paths (manifest: Types.ProductManifest.File[]) = task {
+let updateProduct downloader paths (manifest: Types.ProductManifest.File[]) = task {
     let manifestMap =
         manifest
-        |> Array.map (fun file -> normalizeManifestPartialPath file.Path, file)
+        |> Array.map (fun file -> Product.normalizeManifestPartialPath file.Path, file)
         |> Map.ofArray
-    let getFileHash file =
-        match SHA1.hashFile file |> Result.map Hex.toString with
-        | Ok hash -> Some (hash.ToLower())
+    
+    let tryGenHash file =
+        match Product.generateFileHashStr Product.hashFile file with
+        | Ok hash -> Some hash
         | Error e ->
            Log.warn $"Unable to get hash of file '%s{file}' - %s{e.ToString()}"
            None
-    let parseHashCache hashMapPath =
-        if File.Exists(hashMapPath) then
-            FileIO.readAllLines hashMapPath
-            |> Task.mapResult (fun (lines: string[]) ->
-                lines
-                |> Array.choose (fun line ->
-                    let parts = line.Split("|", StringSplitOptions.RemoveEmptyEntries)
-                    if parts.Length = 2 then
-                        Some (parts.[0], parts.[1])
-                    else
-                        None)
-                |> Map.ofArray)
-        else Map.empty |> Ok |> Task.fromResult
-           
-    let getFileHashes cache dir (filePaths: string seq) =
-        filePaths
-        |> Seq.filter File.Exists
-        |> Seq.map (fun file -> file.Replace(dir, "").TrimStart(Path.DirectorySeparatorChar))
-        |> Seq.filter manifestMap.ContainsKey
-        |> Seq.choose (fun file ->
-            cache
-            |> Map.tryFind file
-            |> Option.orElseWith (fun () -> getFileHash (Path.Combine(dir, file)))
-            |> Option.map (fun hash -> (file, hash)))
-        |> Map.ofSeq
 
-    let verifyFiles files =
-        let invalidFiles = files |> Seq.filter (fun (_, _, valid) -> not valid) |> Seq.map (fun (path, _, _) -> path)
+    let verifyFiles (files: Http.FileDownloadResponse[]) =
+        let invalidFiles = files |> Seq.filter (fun file -> file.Integrity = Http.Invalid) |> Seq.map (fun file -> file.FilePath)
         if Seq.isEmpty invalidFiles then Ok ()
         else invalidFiles |> String.join Environment.NewLine |> Error
-    
-    let progress = Progress<DownloadProgress>(fun p ->
-        let total = p.TotalBytes |> Int64.toFriendlyByteString
-        let percent = float p.BytesSoFar / float p.TotalBytes
-        Console.Write($"\rDownloading %d{p.TotalFiles} files (%s{total}) - {percent:P0}"))
 
-    let writeHashCache append path hashMap = task {
-        let writeAllLines = if append then FileIO.appendAllLines else FileIO.writeAllLines
-        let! write =
-            hashMap
-            |> Map.toSeq
-            |> Seq.map (fun (file, hash) -> $"%s{file}|%s{hash}")
-            |> writeAllLines path
-        match write with
+    let writeHashCache append path hashMap = task { 
+        match! Product.writeHashCache append path hashMap with
         | Ok () -> Log.debug $"Wrote hash cache to '%s{path}'"
-        | Error e -> Log.warn $"Unable to write hash cache at '%s{paths.ProductHashMap}' - %s{e}" }
+        | Error e -> Log.warn $"Unable to write hash cache at '%s{path}' - %s{e}" }
     
+    let getFileHashes = Product.getFileHashes tryGenHash File.Exists (manifestMap |> Map.keys)
     let processFiles productHashMap cacheHashMap =
         paths.ProductCacheDir
         |> FileIO.ensureDirExists
         |> Result.map (fun cacheDir ->
+            Log.info "Determining which files need to be updated. This may take a while."
             let cachedHashes = getFileHashes cacheHashMap cacheDir (Directory.EnumerateFiles(cacheDir, "*.*", SearchOption.AllDirectories))
             let validCachedFiles = cachedHashes |> Map.filter (fun file hash -> manifestMap.[file].Hash = hash) |> Map.keys
             let manifestKeys = manifestMap |> Map.keys
@@ -303,15 +233,18 @@ let updateProduct (httpClient: HttpClient) (throttler: SemaphoreSlim) cancellati
             |> Seq.map (fun file -> Map.find file manifestMap)
             |> Seq.toArray, productHashes, cachedHashes)
     
-    Log.info "Determining which files need to be updated. This may take a while."
+    let downloadFiles downloader cacheDir (files: Types.ProductManifest.File[]) =
+        Log.info $"Downloading %d{files.Length} files"
+        Product.downloadFiles downloader cacheDir files
+
     let! cacheHashes = task {
-        match! parseHashCache paths.CacheHashMap with
+        match! Product.parseHashCache paths.CacheHashMap with
         | Ok hashes -> return hashes
         | Error e ->
             Log.warn $"Unable to parse hash map at '%s{paths.CacheHashMap}' - %s{e}"
             return Map.empty }
     let! productHashes = task {
-        match! parseHashCache paths.ProductHashMap with
+        match! Product.parseHashCache paths.ProductHashMap with
         | Ok hashes -> return hashes
         | Error e ->
             Log.warn $"Unable to parse hash map at '%s{paths.ProductHashMap}' - %s{e}"
@@ -324,14 +257,14 @@ let updateProduct (httpClient: HttpClient) (throttler: SemaphoreSlim) cancellati
             do! write paths.ProductHashMap productHashes
             do! write paths.CacheHashMap cacheHashes
             return Ok invalidFiles })
-        |> Task.bindTaskResult (downloadFiles httpClient throttler paths.ProductCacheDir progress cancellationToken)
+        |> Task.bindTaskResult (downloadFiles downloader paths.ProductCacheDir)
         |> Task.bindTaskResult (fun files -> task {
             printfn ""
             do!
                 files
-                |> Seq.map (fun (path, hash, _) ->
+                |> Seq.map (fun response ->
                     let trim = paths.ProductCacheDir |> String.ensureEndsWith Path.DirectorySeparatorChar
-                    path.Replace(trim, ""), hash)
+                    response.FilePath.Replace(trim, ""), response.Hash)
                 |> Map.ofSeq
                 |> writeHashCache true paths.ProductHashMap
             return verifyFiles files }) }
@@ -428,21 +361,28 @@ let run settings cancellationToken = task {
                         tmpClient.Timeout <- TimeSpan.FromMinutes(5.)                        
                         let tmp = products |> filterProducts (fun p -> match p with | Playable p -> Some p | _ -> None)
                         let! asdf = Api.getProductManifest tmpClient (Uri("http://cdn.zaonce.net/elitedangerous/win/manifests/Win64_Release_3_7_7_500+%282021.01.28.254828%29.xml.gz"))
-                        //let! fdsa = Api.getProductManifest httpClient (Uri("http://cdn.zaonce.net/elitedangerous/win/manifests/Win64_4_0_0_10_Alpha+%282021.04.09.263090%29.xml.gz"))
                         do! match asdf with
                             | Ok man -> task {
                                 let p = tmp.[0]
                                 Log.info $"Updating %s{p.Name}"
                                 let productsDir = Path.Combine(settings.CbLauncherDir, "Products")
                                 let productDir = Path.Combine(productsDir, p.Directory)
-                                use throttler = new SemaphoreSlim(4, 4)
                                 
                                 let productCacheDir = Path.Combine(Environment.cacheDir, $"%s{man.Title}%s{man.Version}")
                                 let pathInfo = { ProductDir = productDir
                                                  ProductCacheDir = productCacheDir
                                                  CacheHashMap = Path.Combine(productCacheDir, "hashmap.txt")
                                                  ProductHashMap = Path.Combine(Environment.cacheDir, $"hashmap.%s{Path.GetFileName(productDir)}.txt") }
-                                match! updateProduct tmpClient throttler cancellationToken pathInfo man.Files with
+                                let progress = Progress<DownloadProgress>(fun p ->
+                                    let total = p.TotalBytes |> Int64.toFriendlyByteString
+                                    let percent = float p.BytesSoFar / float p.TotalBytes
+                                    Console.Write($"\rDownloading %d{p.TotalFiles} files (%s{total}) - {percent:P0}")) :> IProgress<DownloadProgress>
+
+                                use semaphore = new SemaphoreSlim(4, 4)
+                                use sha1 = SHA1.Create()
+                                let throttled progress = throttledAction semaphore (downloadFile tmpClient sha1 cancellationToken progress)
+                                let downloader = { Download = throttled; Progress = progress }
+                                match! updateProduct downloader pathInfo man.Files with
                                 | Ok () ->
                                     Log.info $"Finished downloading update for %s{p.Name}"
                                     File.Delete(pathInfo.CacheHashMap)
@@ -451,7 +391,7 @@ let run settings cancellationToken = task {
                                 | Error e -> Log.error $"Unable to download update for %s{p.Name} - %s{e}" }
                             | Error e -> () |> Task.fromResult
 
-                        let! productManifestTasks =
+                        let! productManifests =
                             productsToUpdate
                             |> Array.map (fun p ->
                                 p.Metadata
@@ -459,11 +399,11 @@ let run settings cancellationToken = task {
                                 |> Option.defaultValue (Task.FromResult(Error $"No metadata for %s{p.Name}")))
                             |> Task.whenAll
                         
-                        let productManifests =
-                            productManifestTasks
+                        let productsToUpdate =
+                            productManifests
                             |> Array.zip productsToUpdate
                             |> Array.choose (fun (_, manifest) -> match manifest with Ok m -> Some m | Error _ -> None)
-                        let failedManifests = productManifestTasks |> Array.choose (function Ok _ -> None | Error e -> Some e)
+                        let failedManifests = productManifests |> Array.choose (function Ok _ -> None | Error e -> Some e)
                         
                         let playableProducts = products |> filterProducts (fun p -> match p with | Playable p -> Some p | _ -> None)
                         let selectedProduct =

src/MinEdLauncher/Extensions.fs

@@ -47,7 +47,10 @@ module Result =
         
 
 module Seq =
+    open System.Linq
+    
     let chooseResult r = r |> Seq.choose (fun r -> match r with | Error _ -> None | Ok v -> Some v)
+    let intersect (itemsToInclude: seq<'T>) (source: seq<'T>) = source.Intersect(itemsToInclude)
     
 module Map =
     // https://stackoverflow.com/a/50925864/182821

src/MinEdLauncher/Http.fs

@@ -0,0 +1,29 @@
+module MinEdLauncher.Http
+
+open System
+open System.IO
+open System.Security.Cryptography
+open System.Net.Http
+open System.Threading.Tasks
+open FSharp.Control.Tasks.NonAffine
+open HttpClientExtensions
+
+type DownloadProgress = { TotalFiles: int; BytesSoFar: int64; TotalBytes: int64; }
+type DownloadAll<'a, 'b> = IProgress<int> -> 'a[] -> Task<'b[]>
+type Downloader<'a, 'b> = { Download: DownloadAll<'a, 'b>; Progress: IProgress<DownloadProgress> }
+type FileDownloadRequest = { RemotePath: string; TargetPath: string; ExpectedHash: string }
+type FileIntegrity = Valid | Invalid
+module FileIntegrity =
+    let fromBool = function true -> Valid | false -> Invalid
+type FileDownloadResponse = { FilePath: string; Hash: string; Integrity: FileIntegrity }
+
+let downloadFile (httpClient: HttpClient) (hashAlgorithm: HashAlgorithm) cancellationToken progress request = task {
+    let bufferSize = 8192
+    use fileStream = new FileStream(request.TargetPath, FileMode.Create, FileAccess.Write, FileShare.Write, bufferSize, FileOptions.Asynchronous)
+    use cryptoStream = new CryptoStream(fileStream, hashAlgorithm, CryptoStreamMode.Write) // Calculate hash as file is downloaded
+    
+    do! httpClient.DownloadAsync(request.RemotePath, cryptoStream, bufferSize, progress, cancellationToken)
+    cryptoStream.Dispose()
+    let hash = hashAlgorithm.Hash |> Hex.toString |> String.toLower
+    return { FilePath = request.TargetPath; Hash = hash; Integrity = request.ExpectedHash = hash |> FileIntegrity.fromBool } }
+

src/MinEdLauncher/MinEdLauncher.fsproj

@@ -41,6 +41,7 @@
     <Compile Include="Token.fs" />
     <Compile Include="Types.fs" />
     <Compile Include="Interop.fs" />
+    <Compile Include="Http.fs" />
     <Compile Include="Product.fs" />
     <Compile Include="Process.fs" />
     <Compile Include="Steam.fs" />

src/MinEdLauncher/Product.fs

@@ -4,9 +4,104 @@ open System
 open System.Diagnostics
 open System.IO
 open System.Runtime.InteropServices
+open System.Threading
+open System.Threading.Tasks
+open FSharp.Control.Tasks.NonAffine
+open MinEdLauncher.Http
 open MinEdLauncher.Rop
 open MinEdLauncher.Types
 
+let generateFileHashStr (hashFile: string -> Result<byte[], 'TError>) (file: string) =
+    hashFile file
+    |> Result.map (Hex.toString >> String.toLower)
+
+let hashFile = SHA1.hashFile
+
+let private mapHashPair file hash = (file, hash)
+let private getFileRelativeDirectory relativeTo (file: string) = file.Replace(relativeTo, "").TrimStart(Path.DirectorySeparatorChar)
+
+let private generateFileHashes tryGenHash productDir (manifestFiles: string Set) (filePaths: string seq) =
+    let tryGetHash file =
+        let getFileAbsoluteDirectory file = Path.Combine(productDir, file)
+        tryGenHash (getFileAbsoluteDirectory file) |> Option.map (mapHashPair file)
+        
+    filePaths
+    |> Seq.map (getFileRelativeDirectory productDir)
+    |> Seq.intersect manifestFiles
+    |> Seq.choose tryGetHash
+    |> Map.ofSeq
+
+let getFileHashes tryGenHash fileExists (manifestFiles: string Set) cache productDir (filePaths: string seq) =
+    let getHashFromCache cache file =
+        cache
+        |> Map.tryFind (getFileRelativeDirectory productDir file)
+        |> Option.map (mapHashPair file)
+
+    let cachedHashes = filePaths |> Seq.choose (getHashFromCache cache)
+    let missingHashes = filePaths |> Seq.except (cachedHashes |> Seq.map fst) |> Seq.filter fileExists
+    generateFileHashes tryGenHash productDir manifestFiles missingHashes
+
+let parseHashCacheLines (lines: string seq) =
+    lines
+    |> Seq.choose (fun line ->
+        let parts = line.Split("|", StringSplitOptions.RemoveEmptyEntries)
+        if parts.Length = 2 then
+            Some (parts.[0], parts.[1])
+        else
+            None)
+    |> Map.ofSeq
+    
+let parseHashCache filePath =
+    if File.Exists(filePath) then
+        FileIO.readAllLines filePath
+        |> Task.mapResult parseHashCacheLines
+    else Map.empty |> Ok |> Task.fromResult
+
+let mapHashMapToLines hashMap =
+    hashMap
+    |> Map.toSeq
+    |> Seq.map (fun (file, hash) -> $"%s{file}|%s{hash}")
+    
+let writeHashCache append path hashMap =
+    let writeAllLines = if append then FileIO.appendAllLines else FileIO.writeAllLines
+    mapHashMapToLines hashMap |> writeAllLines path
+
+let normalizeManifestPartialPath (path: string) =
+    if not (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) then
+        path.Replace('\\', '/')
+    else
+        path
+
+let mapFileToRequest destDir (file: Types.ProductManifest.File) =
+    let path = normalizeManifestPartialPath file.Path
+    let targetPath = Path.Combine(destDir, path)
+    { RemotePath = file.Download; TargetPath = targetPath; ExpectedHash = file.Hash }
+
+let downloadFiles downloader destDir (files: Types.ProductManifest.File[]) : Task<Result<FileDownloadResponse[], string>> = task {
+    let combinedTotalBytes = files |> Seq.sumBy (fun f -> int64 f.Size)
+    let combinedBytesSoFar = ref 0L
+    
+    let relativeProgress = Progress<int>(fun bytesRead ->
+        let bytesSoFar = Interlocked.Add(combinedBytesSoFar, int64 bytesRead)
+        downloader.Progress.Report({ TotalFiles = files.Length
+                                     BytesSoFar = bytesSoFar
+                                     TotalBytes = combinedTotalBytes })) :> IProgress<int>
+        
+    let ensureDirectories requests =
+        requests
+        |> Seq.iter (fun request ->
+            let dirName = Path.GetDirectoryName(request.TargetPath);
+            if dirName.Length > 0 then
+                Directory.CreateDirectory(dirName) |> ignore )
+        
+    let requests = files |> Array.map (mapFileToRequest destDir)
+    
+    try
+        ensureDirectories requests
+        let! result = downloader.Download relativeProgress requests
+        return Ok result
+    with e -> return e.ToString() |> Error }
+
 let createArgString vr (lang: string option) edSession machineId timestamp watchForCrashes platform hashFile (product:ProductDetails) =
     let targetOptions = String.Join(" ", [
         if lang.IsSome then "/language " + lang.Value 

tests/Product.fs

@@ -1,6 +1,9 @@
 module MinEdLauncher.Tests.Product
 
+open System
 open System.IO
+open MinEdLauncher
+open MinEdLauncher.Http
 open MinEdLauncher.Product
 open MinEdLauncher.Token
 open MinEdLauncher.Types
@@ -11,6 +14,9 @@ open Expecto
           if (subject.Contains(substring)) then
             failtestf "%s. Expected subject string '%s' to not contain substring '%s'."
                             message subject substring
+        let stringEqual (actual: string) (expected: string) comparisonType message =
+            if not (String.Equals(actual, expected, comparisonType)) then
+                failtest $"%s{message}. Actual value was %s{actual} but had expected it to be %s{expected}."
 
     [<Tests>]
     let tests =
@@ -189,6 +195,151 @@ open Expecto
                     Expect.equal info.WorkingDirectory product.WorkingDir.FullName ""
                 }
             ]
-    //        testProperty "Unknown arg doesn't change any values" <|
-    //            fun (args:string[]) -> parse args = Settings.defaults
+            testList "generateFileHashStr" [
+                test "converts to hex representation" {
+                    let hashFile = (fun _ -> Result.Ok [| 10uy; 2uy; 15uy; 11uy |])
+                    let expected = "0A020F0B"
+                    
+                    let result = generateFileHashStr hashFile "" |> Result.defaultValue ""
+                    
+                    Expect.stringEqual result expected StringComparison.OrdinalIgnoreCase ""
+                }
+                test "converts to all lowercase string" {
+                    let hashFile = (fun _ -> Result.Ok [| 10uy; 2uy; 15uy; 11uy |])
+                    
+                    let result = (generateFileHashStr hashFile "") |> Result.defaultValue ""
+                    
+                    Expect.all result (fun c -> Char.IsDigit(c) || Char.IsLower(c)) ""
+                } ]
+            testList "getFileHashes" [
+                test "skips files that don't exist" {
+                    let tryGenHash = (fun _ -> Some "hash")
+                    let fileExists = (fun _ -> false)
+                    let baseDir = Path.Combine("the", "directory")
+                    let manifestFiles = [ Path.Combine("file", "path") ] |> Set.ofList
+                    let cache = Map.empty<string, string> 
+                    let filePaths = [ Path.Combine(baseDir, "file", "path") ]
+                    
+                    let result = getFileHashes tryGenHash fileExists manifestFiles cache baseDir filePaths
+                    
+                    Expect.isEmpty result ""
+                }
+                test "tries to get hash of absolute path" {
+                    let baseDir = Path.Combine("the", "directory")
+                    let absolutePath = Path.Combine(baseDir, "file", "path")
+                    let tryGenHash = (fun path -> if path = absolutePath then Some "hash" else None)
+                    let fileExists = (fun _ -> true)
+                    let manifestFiles = [ Path.Combine("file", "path") ] |> Set.ofList
+                    let cache = Map.empty<string, string> 
+                    let filePaths = [ absolutePath ]
+                    
+                    let result = getFileHashes tryGenHash fileExists manifestFiles cache baseDir filePaths
+                    
+                    Expect.hasLength result filePaths.Length ""
+                }
+                test "ignores files not in manifest" {
+                    let tryGenHash = (fun _ -> Some "hash")
+                    let fileExists = (fun _ -> true)
+                    let baseDir = Path.Combine("the", "directory")
+                    let manifestFile = Path.Combine("manifest", "file")
+                    let nonManifestFile = Path.Combine("nonmanifest", "file")
+                    let manifestFiles = [ manifestFile ] |> Set.ofList
+                    let cache = Map.empty<string, string> 
+                    let filePaths = [ manifestFile; nonManifestFile ] |> List.map (fun path -> Path.Combine(baseDir, path))
+                    
+                    let result = getFileHashes tryGenHash fileExists manifestFiles cache baseDir filePaths
+                    
+                    Expect.hasLength result 1 ""
+                }
+                test "uses relative path to check manifest" {
+                    let tryGenHash = (fun _ -> Some "hash")
+                    let fileExists = (fun _ -> true)
+                    let baseDir = Path.Combine("the", "directory")
+                    let manifestFile = Path.Combine("manifest", "file")
+                    let manifestFiles = [ manifestFile ] |> Set.ofList
+                    let cache = Map.empty<string, string> 
+                    let filePaths = [ Path.Combine(baseDir, manifestFile) ]
+                    
+                    let result = getFileHashes tryGenHash fileExists manifestFiles cache baseDir filePaths
+                    
+                    Expect.hasLength result 1 ""
+                }
+                test "skips files that weren't able to generate a hash" {
+                    let tryGenHash = (fun _ -> None)
+                    let fileExists = (fun _ -> true)
+                    let baseDir = Path.Combine("the", "directory")
+                    let manifestFiles = [ Path.Combine("file", "path") ] |> Set.ofList
+                    let cache = Map.empty<string, string> 
+                    let filePaths = [ Path.Combine(baseDir, "file", "path") ]
+                    
+                    let result = getFileHashes tryGenHash fileExists manifestFiles cache baseDir filePaths
+                    
+                    Expect.isEmpty result ""
+                }
+                test "skips files if they are cached" {
+                    let tryGenHash = (fun _ -> failtest "Shouldn't try to hash file")
+                    let fileExists = (fun _ -> false)
+                    let baseDir = Path.Combine("the", "directory")
+                    let manifestFile = Path.Combine("manifest", "file")
+                    let manifestFiles = [ manifestFile ] |> Set.ofList
+                    let cache = [ (manifestFile, "hash") ] |> Map.ofList
+                    let filePaths = [ Path.Combine(baseDir, manifestFile) ]
+                    
+                    let result = getFileHashes tryGenHash fileExists manifestFiles cache baseDir filePaths
+                    
+                    Expect.hasLength result 0 ""
+                }
+            ]
+            testList "parseHashCacheLines" [
+                test "skips if line has fewer than two parts" {
+                    let lines = seq { "path"; "path|" }
+                    
+                    let result = parseHashCacheLines lines
+                    
+                    Expect.isEmpty result ""
+                }
+                test "skips if lines has more than two parts" {
+                    let lines = seq { "path|hash|something" }
+                    
+                    let result = parseHashCacheLines lines
+                    
+                    Expect.isEmpty result ""
+                }
+                test "can parse valid line" {
+                    let lines = seq { "path|hash" }
+                    let expected = [ ("path", "hash") ] |> Map.ofList
+                    
+                    let result = parseHashCacheLines lines
+                    
+                    Expect.equal result expected ""
+                } ]
+            testList "mapHashMapToLines" [
+                test "can parse valid line" {
+                    let map = [ ("path", "hash") ] |> Map.ofList
+                    let expected = seq { "path|hash" }
+                    
+                    let result = mapHashMapToLines map
+                    
+                    Expect.sequenceEqual result expected ""
+                } ]
+            testList "normalizeManifestPartialPath" [
+                test "results in correct path separator" {
+                    let path = "a\\windows\\dir"
+                    let expected = Path.Combine("a", "windows", "dir")
+                    let result = normalizeManifestPartialPath path
+                    
+                    Expect.equal result expected ""
+                } ]
+            testList "mapFileToRequest" [
+                test "maps correctly" {
+                    let hash = "hash"
+                    let remotePath = "http://remote.path"
+                    let destDir = "dest"
+                    let file = ProductManifest.File("a\\windows\\dir", hash, 0, remotePath)
+                    let expected = { RemotePath = remotePath; TargetPath = Path.Combine(destDir, "a", "windows", "dir"); ExpectedHash = hash }
+                    
+                    let result = mapFileToRequest destDir file
+                    
+                    Expect.equal result expected ""
+                } ]
         ]