diff --git a/src/MinEdLauncher/Api.fs b/src/MinEdLauncher/Api.fs index 4fb9c9e..d44a648 100644 --- a/src/MinEdLauncher/Api.fs +++ b/src/MinEdLauncher/Api.fs @@ -1,6 +1,7 @@ module MinEdLauncher.Api open System +open System.IO.Compression open System.Net open System.Net.Http open System.Text @@ -254,6 +255,14 @@ let getAuthorizedProducts platform lang connection = task { |> Result.map (fun products -> products |> Seq.sortBy (fun p -> p.SortKey) |> List.ofSeq) } +let getProductManifest (httpClient: HttpClient) (uri: Uri) = task { + try + use! responseStream = httpClient.GetStreamAsync(uri) + use decompStream = new GZipStream(responseStream, CompressionMode.Decompress) + return ProductManifest.Load(decompStream) |> Ok + with e -> return e.ToString() |> Error +} + let checkForUpdate platform machineId connection product = task { match product with | Unknown name -> return Error $"{name}: Can't check updates for unknown product" @@ -270,14 +279,26 @@ let checkForUpdate platform machineId connection product = task { use request = buildRequest "/3.0/user/installer" platform connection queryParams let! content = fetch connection.HttpClient request - return content - >>= Json.parseProp "version" - >>= Json.asVersion - |> Result.map (fun remoteVersion -> - if remoteVersion = product.Version then - product |> Playable + let version = content >>= Json.parseProp "version" >>= Json.asVersion + let remotePath = content >>= Json.parseProp "remotePath" >>= Json.toString |> (Result.map Hex.parseIso88591String) + let localFile = content >>= Json.parseProp "localFile" >>= Json.toString |> (Result.map System.IO.Path.GetFileName) + let hash = content >>= Json.parseProp "md5" >>= Json.toString + let size = content >>= Json.parseProp "size" >>= Json.toInt64 + + return + match version, remotePath, localFile, hash, size with + | Ok version, Ok remotePath, Ok localFile, Ok hash, Ok size -> + let metadata = { Hash = hash; LocalFile = localFile; RemotePath = Uri(remotePath); Size = size; Version = version } + let product = { product with Metadata = Some metadata } + if version = product.Version then + product |> Playable |> Ok else - product |> RequiresUpdate) + product |> RequiresUpdate |> Ok + | _ -> + let content = content >>= Json.toString |> Result.defaultWith id + let msg = $"Unexpected json object %s{content}" + Log.debug msg + Error msg } let checkForUpdates platform machineId connection (products: Product list) = task { diff --git a/src/MinEdLauncher/App.fs b/src/MinEdLauncher/App.fs index ee7fa18..758575e 100644 --- a/src/MinEdLauncher/App.fs +++ b/src/MinEdLauncher/App.fs @@ -1,13 +1,18 @@ module MinEdLauncher.App open System.IO +open System.Net.Http open System.Runtime.InteropServices +open System.Security.Cryptography +open System.Threading open MinEdLauncher open MinEdLauncher.Token open FSharp.Control.Tasks.NonAffine open System open System.Diagnostics +open System.Threading.Tasks open MinEdLauncher.Types +open MinEdLauncher.HttpClientExtensions type LoginResult = | Success of Api.Connection @@ -111,7 +116,7 @@ let rec launchProduct proton processArgs restart productName product = | Product.RunResult.AlreadyRunning -> Log.info $"%s{productName} is already running" | Product.RunResult.Error e -> Log.error $"Couldn't start selected product: %s{e.ToString()}" -let promptForProduct (products: ProductDetails array) = +let promptForProductToPlay (products: ProductDetails array) (cancellationToken:CancellationToken) = printfn $"Select a product to launch (default=1):" products |> Array.indexed @@ -119,14 +124,16 @@ let promptForProduct (products: ProductDetails array) = let rec readInput() = printf "Product: " - let userInput = Console.ReadKey() + let userInput = Console.ReadKey(true) printfn "" let couldParse, index = if userInput.Key = ConsoleKey.Enter then true, 1 else Int32.TryParse(userInput.KeyChar.ToString()) - if couldParse && index > 0 && index < products.Length then + if cancellationToken.IsCancellationRequested then + None + else if couldParse && index > 0 && index < products.Length then let product = products.[index - 1] let filters = String.Join(", ", product.Filters) Log.debug $"User selected %s{product.Name} - %s{product.Sku} - %s{filters}" @@ -136,7 +143,132 @@ let promptForProduct (products: ProductDetails array) = readInput() readInput() -let run settings = task { +let promptForProductsToUpdate (products: ProductDetails array) = + printfn $"Select product(s) to update (eg: \"1\", \"1 2 3\") (default=None):" + products + |> Array.indexed + |> Array.iter (fun (i, product) -> printfn $"%i{i + 1}) %s{product.Name}") + + let rec readInput() = + let userInput = Console.ReadLine() + + if String.IsNullOrWhiteSpace(userInput) then + [||] + else + let selection = + userInput + |> Regex.split @"\D+" + |> Array.choose (fun d -> + if String.IsNullOrEmpty(d) then + None + else + match Int32.Parse(d) with + | n when n > 0 && n < products.Length -> Some n + | _ -> None) + |> Array.map (fun i -> products.[i - 1]) + if selection.Length > 0 then + selection + else + printfn "Invalid selection" + readInput() + readInput() + +let normalizeManifestPartialPath (path: string) = + if not (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) then + path.Replace('\\', '/') + else + path + +type DownloadProgress = { TotalFiles: int; BytesSoFar: int64; TotalBytes: int64; } +let downloadFiles (httpClient: HttpClient) destDir (progress: IProgress) cancellationToken (files: Types.ProductManifest.File[]) = + let combinedTotalBytes = files |> Seq.sumBy (fun f -> int64 f.Size) + let combinedBytesSoFar = ref 0L + let downloadFile (file: Types.ProductManifest.File) = task { + let path = normalizeManifestPartialPath file.Path + let dest = Path.Combine(destDir, path) + + let dirName = Path.GetDirectoryName(dest); + if dirName.Length > 0 then + Directory.CreateDirectory(dirName) |> ignore + + use sha1 = SHA1.Create() + use fileStream = new FileStream(dest, FileMode.Create, FileAccess.Write, FileShare.Write, 4096, FileOptions.Asynchronous) + use cryptoStream = new CryptoStream(fileStream, sha1, CryptoStreamMode.Write) // Calculate hash as file is downloaded + let totalReads = ref 0L + let relativeProgress = Progress(fun bytesRead -> + let bytesSoFar = Interlocked.Add(combinedBytesSoFar, int64 bytesRead) + let totalReads = Interlocked.Increment(totalReads) // Hack so that the console isn't written to too fast + if totalReads % 100L = 0L then + progress.Report({ TotalFiles = files.Length + BytesSoFar = bytesSoFar + TotalBytes = combinedTotalBytes })) + do! httpClient.DownloadAsync(file.Download, cryptoStream, relativeProgress, cancellationToken) + cryptoStream.Dispose() + let hash = sha1.Hash |> Hex.toString |> String.toLower + return dest, file.Hash = hash } + + try + files + |> Array.map downloadFile + |> Task.whenAll + |> Ok + with e -> e.ToString() |> Error + +let updateProduct (httpClient: HttpClient) productDir cacheDir (product: ProductDetails) cancellationToken (manifest: Types.ProductManifest.Manifest) = + let manifestMap = + manifest.Files + |> Array.map (fun file -> normalizeManifestPartialPath file.Path, file) + |> Map.ofArray + let getFileHash file = + match SHA1.hashFile file |> Result.map Hex.toString with + | Ok hash -> Some (hash.ToLower()) + | Error e -> + Log.warn $"Unable to get hash of file '%s{file}' - %s{e.ToString()}" + None + let getFileHashes dir = + Directory.EnumerateFiles(dir, "*.*", SearchOption.AllDirectories) + |> Seq.map (fun file -> file.Replace(dir, "").TrimStart(Path.DirectorySeparatorChar)) + |> Seq.filter manifestMap.ContainsKey + |> Seq.choose (fun file -> getFileHash (Path.Combine(dir, file)) |> Option.map (fun hash -> (file, hash))) + |> Map.ofSeq + + let verifyFiles files = + let invalidFiles = files |> Seq.filter (fun (path, valid) -> not valid) |> Seq.map fst + if Seq.isEmpty invalidFiles then Ok () + else invalidFiles |> String.join Environment.NewLine |> Error + + let progress = Progress(fun p -> + let total = p.TotalBytes |> Int64.toFriendlyByteString + let percent = float p.BytesSoFar / float p.TotalBytes + Console.SetCursorPosition(0, Console.CursorTop) + Console.Write($"Downloading %d{p.TotalFiles} files (%s{total}) - {percent:P0}")) + + let findInvalidFiles() = + cacheDir + |> FileIO.ensureDirExists + |> Result.map (fun cacheDir -> + let cachedHashes = getFileHashes cacheDir + let validCachedFiles = cachedHashes |> Map.filter (fun file hash -> manifestMap.[file].Hash = hash) |> Map.keys + + manifestMap + |> Map.keys + |> Seq.except validCachedFiles + |> Seq.choose (fun file -> + let fullPath = Path.Combine(productDir, file) + + if File.Exists(fullPath) then + let hash = getFileHash fullPath + hash |> Option.filter (fun hash -> manifestMap.[file].Hash <> hash) + else Some file) + |> Seq.map (fun file -> Map.find file manifestMap) + |> Seq.toArray) + + Log.info "Determining which files need to be updated. This may take a while." + findInvalidFiles() + |> Result.bind (downloadFiles httpClient cacheDir progress cancellationToken) + |> Result.mapTask verifyFiles + +let run settings cancellationToken = task { if RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && settings.Platform = Steam then Steam.fixLcAll() @@ -173,8 +305,6 @@ let run settings = task { #else MachineId.getWineId() #endif - // TODO: Check if launcher version is compatible with current ED version - match machineId with | Ok machineId -> let lang = settings.PreferredLanguage |> Option.defaultValue "en" @@ -207,11 +337,62 @@ let run settings = task { | [] -> "None" | p -> String.Join(Environment.NewLine + "\t", p) Log.info $"Available Products:{Environment.NewLine}\t%s{availableProductsDisplay}" - let playableProducts = + let filterProducts f products = products |> Result.defaultValue [] - |> List.choose (fun p -> match p with | Playable p -> Some p | _ -> None) + |> List.choose f |> List.toArray + let productsRequiringUpdate = products |> filterProducts (fun p -> match p with | RequiresUpdate p -> Some p | _ -> None) + + let productsToUpdate = + let products = + if true(*settings.AutoUpdate*) then + productsRequiringUpdate + else + productsRequiringUpdate |> promptForProductsToUpdate + products + |> Array.filter (fun p -> p.Metadata.IsNone) + |> Array.iter (fun p -> Log.error $"Unknown product metadata for %s{p.Name}") + + products |> Array.filter (fun p -> p.Metadata.IsSome) + +// use tmpClient = new HttpClient() +// tmpClient.Timeout <- TimeSpan.FromMinutes(5.) +// let tmp = products |> filterProducts (fun p -> match p with | Playable p -> Some p | _ -> None) +// let! asdf = Api.getProductManifest tmpClient (Uri("http://cdn.zaonce.net/elitedangerous/win/manifests/Win64_Release_3_7_7_500+%282021.01.28.254828%29.xml.gz")) +// //let! fdsa = Api.getProductManifest httpClient (Uri("http://cdn.zaonce.net/elitedangerous/win/manifests/Win64_4_0_0_10_Alpha+%282021.04.09.263090%29.xml.gz")) +// do! match asdf with +// | Ok man -> task { +// let p = tmp.[0] +// Log.info $"Updating %s{p.Name}" +// let productsDir = Path.Combine(settings.CbLauncherDir, "Products") +// let productDir = Path.Combine(productsDir, p.Directory) +// let cacheDir = Path.Combine(productsDir, $".cache-%s{man.Title}%s{man.Version}") +// let! result = updateProduct tmpClient productDir cacheDir p cancellationToken man +// printfn "" +// match result with +// | Ok () -> +// Log.info $"Finished downloading update for %s{p.Name}" +// FileIO.mergeDirectories productDir cacheDir +// Log.debug $"Moved downloaded files from '%s{cacheDir}' to '%s{productDir}'" +// | Error e -> Log.error $"Unable to download update for %s{p.Name} - %s{e}" } +// | Error e -> () |> Task.fromResult + + let! productManifestTasks = + productsToUpdate + |> Array.map (fun p -> + p.Metadata + |> Option.map (fun m -> Api.getProductManifest httpClient m.RemotePath) + |> Option.defaultValue (Task.FromResult(Error $"No metadata for %s{p.Name}"))) + |> Task.whenAll + + let productManifests = + productManifestTasks + |> Array.zip productsToUpdate + |> Array.choose (fun (_, manifest) -> match manifest with Ok m -> Some m | Error _ -> None) + let failedManifests = productManifestTasks |> Array.choose (function Ok _ -> None | Error e -> Some e) + + let playableProducts = products |> filterProducts (fun p -> match p with | Playable p -> Some p | _ -> None) let selectedProduct = if settings.AutoRun then playableProducts @@ -219,11 +400,12 @@ let run settings = task { || p.Filters |> Set.union settings.ProductWhitelist |> Set.count > 0) |> Array.tryHead else if playableProducts.Length > 0 then - promptForProduct playableProducts + promptForProductToPlay playableProducts cancellationToken else None - match selectedProduct, true with - | Some product, true -> + match selectedProduct, cancellationToken.IsCancellationRequested with + | _, true -> () + | Some product, _ -> let gameLanguage = Cobra.getGameLang settings.CbLauncherDir settings.PreferredLanguage let processArgs() = Product.createArgString settings.DisplayMode gameLanguage connection.Session machineId (runningTime()) settings.WatchForCrashes settings.Platform SHA1.hashFile product @@ -233,10 +415,9 @@ let run settings = task { launchProduct settings.Proton processArgs settings.Restart product.Name p Process.stopProcesses processes | Error msg -> Log.error $"Couldn't start selected product: %s{msg}" - | None, true -> Log.error "No selected project" - | _, _ -> () + | None, _ -> Log.error "No selected project" - if not settings.AutoQuit then + if not settings.AutoQuit && not cancellationToken.IsCancellationRequested then printfn "Press any key to quit..." Console.ReadKey() |> ignore diff --git a/src/MinEdLauncher/Extensions.fs b/src/MinEdLauncher/Extensions.fs index c8d3c71..f5e9cf7 100644 --- a/src/MinEdLauncher/Extensions.fs +++ b/src/MinEdLauncher/Extensions.fs @@ -20,7 +20,10 @@ module Task = | Error m -> return Error m } -module Result = +module Result = + open FSharp.Control.Tasks.NonAffine + open System.Threading.Tasks + let defaultValue value = function | Ok v -> v | Error _ -> value @@ -30,9 +33,22 @@ module Result = let bindTask f = function | Ok v -> f v | Error v -> Error v |> Task.fromResult + let mapTask f (result: Result, 'TError>) = + match result with + | Ok v -> task { + let! result = v + return f result } + | Error v -> Error v |> Task.fromResult + module Seq = let chooseResult r = r |> Seq.choose (fun r -> match r with | Error _ -> None | Ok v -> Some v) + +module Map = + // https://stackoverflow.com/a/50925864/182821 + let keys<'k, 'v when 'k : comparison> (map : Map<'k, 'v>) = + Map.fold (fun s k _ -> Set.add k s) Set.empty map + module Rop = let (>>=) switchFunction twoTrackInput = Result.bind twoTrackInput switchFunction @@ -96,6 +112,11 @@ module Json = match Int32.TryParse(str) with | true, value -> Ok value | false, _ -> Error $"Unable to convert string to int '%s{str}'" + let toInt64 (prop:JsonElement) = + let str = prop.ToString() + match Int64.TryParse(str) with + | true, value -> Ok value + | false, _ -> Error $"Unable to convert string to long '%s{str}'" let asDateTime (prop:JsonElement) = let str = prop.ToString() match DateTime.TryParse(str) with @@ -164,7 +185,71 @@ module String = open System open System.Collections.Generic let join (separator: string) (values: IEnumerable<'T>) = String.Join(separator, values) + let toLower (str: string) = str.ToLower() +module Int64 = + open System + + let toFriendlyByteString (n: int64) = + let suf = [| "B"; "KB"; "MB"; "GB"; "TB"; "PB"; "EB" |] //Longs run out around EB + if n = 0L then + "0" + suf.[0] + else + let bytes = float (Math.Abs(n)) + let place = Convert.ToInt32(Math.Floor(Math.Log(bytes, float 1024))) + let num = Math.Round(bytes / Math.Pow(float 1024, float place), 1) + (Math.Sign(n) * int num).ToString() + suf.[place]; + +module StreamExtensions = + open System + open System.Threading + open FSharp.Control.Tasks.NonAffine + + type Stream with + member source.CopyToAsync(destination: Stream, bufferSize: int, ?progress: IProgress, ?cancellationToken: CancellationToken) = task { + let cancellationToken = defaultArg cancellationToken CancellationToken.None + if source = null then + raise (ArgumentNullException(nameof source)) + if not source.CanRead then + raise (ArgumentException("Source stream must be readable", nameof source)) + if destination = null then + raise (ArgumentNullException(nameof(destination))); + if not destination.CanWrite then + raise (ArgumentException("Destination stream must be writable", nameof destination)) + if bufferSize < 0 then + raise (ArgumentOutOfRangeException(nameof bufferSize)) + + let buffer = Array.zeroCreate bufferSize + + // Tasks don't support tail call optimization so use a while loop instead of recursion + // https://github.com/crowded/ply/issues/14 + let mutable write = true + while write do + let! bytesRead = source.ReadAsync(buffer, 0, buffer.Length, cancellationToken) + if bytesRead > 0 then + do! destination.WriteAsync(buffer, 0, bytesRead, cancellationToken) + progress |> Option.iter (fun p -> p.Report(bytesRead)) + else + write <- false } + +module HttpClientExtensions = + open StreamExtensions + open FSharp.Control.Tasks.NonAffine + open System + open System.Net.Http + open System.Threading + + type HttpClient with + member client.DownloadAsync(requestUri: string, destination: Stream, ?progress: IProgress, ?cancellationToken: CancellationToken) = task { + let cancellationToken = defaultArg cancellationToken CancellationToken.None + + use! response = client.GetAsync(requestUri, HttpCompletionOption.ResponseHeadersRead) + use! download = response.Content.ReadAsStreamAsync() + + match progress with + | Some progress -> do! download.CopyToAsync(destination, 81920, progress, cancellationToken) + | None -> do! download.CopyToAsync(destination, cancellationToken) } + module SHA1 = open System.Text open System.Security.Cryptography @@ -185,9 +270,17 @@ module SHA1 = module Hex = open System + open System.Text let toString bytes = BitConverter.ToString(bytes).Replace("-","") let toStringTrunc length bytes = BitConverter.ToString(bytes).Replace("-","").Substring(0, length) + let private iso88591GetString (bytes: byte[]) = Encoding.GetEncoding("ISO-8859-1").GetString(bytes) + let parseIso88591String (str: string) = + str + |> Seq.chunkBySize 2 + |> Seq.map (fun chars -> Convert.ToByte(String(chars), 16)) + |> Seq.toArray + |> iso88591GetString module FileIO = open System @@ -259,7 +352,23 @@ module FileIO = | :? PathTooLongException -> Error "Exceeds maximum length" | :? DirectoryNotFoundException -> Error "The specified path is invalid (for example, it is on an unmapped drive)." | :? IOException -> Error "The directory specified by path is a file or the network name is not known." - | :? NotSupportedException -> Error @"Contains a colon character (:) that is not part of a drive label (""C:\"")." + | :? NotSupportedException -> Error @"Contains a colon character (:) that is not part of a drive label (""C:\"")." + + // https://stackoverflow.com/a/2553245/182821 + let mergeDirectories (target: string) (source: string) = + let sourcePath = source.TrimEnd(Path.DirectorySeparatorChar, ' ') + let targetPath = target.TrimEnd(Path.DirectorySeparatorChar, ' ') + Directory.EnumerateFiles(sourcePath, "*", SearchOption.AllDirectories) + |> Seq.groupBy (fun s -> Path.GetDirectoryName(s)) + |> Seq.iter (fun (folder, files) -> + let targetFolder = folder.Replace(sourcePath, targetPath) + Directory.CreateDirectory(targetFolder) |> ignore + files + |> Seq.iter (fun file -> + let targetFile = Path.Combine(targetFolder, Path.GetFileName(file)) + if File.Exists(targetFile) then File.Delete(targetFile) + File.Move(file, targetFile))) + Directory.Delete(source, true); module Console = open System @@ -284,8 +393,8 @@ module Console = module Regex = open System.Text.RegularExpressions - let replace pattern (replacement: string) input = - Regex.Replace(input, pattern, replacement) + let replace pattern (replacement: string) input = Regex.Replace(input, pattern, replacement) + let split pattern input = Regex.Split(input, pattern) module Environment = open System diff --git a/src/MinEdLauncher/MinEdLauncher.fsproj b/src/MinEdLauncher/MinEdLauncher.fsproj index eb1193a..b5855f5 100644 --- a/src/MinEdLauncher/MinEdLauncher.fsproj +++ b/src/MinEdLauncher/MinEdLauncher.fsproj @@ -59,6 +59,7 @@ + diff --git a/src/MinEdLauncher/Product.fs b/src/MinEdLauncher/Product.fs index b231017..de18a7d 100644 --- a/src/MinEdLauncher/Product.fs +++ b/src/MinEdLauncher/Product.fs @@ -78,7 +78,8 @@ let mapProduct productsDir (product:AuthorizedProduct) = Mode = v.Mode Directory = directory GameArgs = product.GameArgs - ServerArgs = serverArgs } + ServerArgs = serverArgs + Metadata = None } | NotFound file -> Log.info $"Disabling '%s{product.Name}'. Unable to find product at '%s{file}'" Missing { Sku = product.Sku diff --git a/src/MinEdLauncher/Program.fs b/src/MinEdLauncher/Program.fs index 0aba3b2..ba3e752 100644 --- a/src/MinEdLauncher/Program.fs +++ b/src/MinEdLauncher/Program.fs @@ -3,6 +3,7 @@ open System open System.IO open System.Reflection +open System.Threading open FsConfig open FSharp.Control.Tasks.NonAffine open Steam @@ -35,13 +36,17 @@ let getSettings args = [] let main argv = async { + use cts = new CancellationTokenSource() + Console.CancelKeyPress.AddHandler (fun s e -> + cts.Cancel() + e.Cancel <- true) try do! Async.SwitchToThreadPool () Log.debug $"Args: %A{argv}" let! settings = getSettings argv |> Async.AwaitTask Log.debug $"Settings: %A{settings}" return! match settings with - | Ok settings -> App.run settings |> Async.AwaitTask + | Ok settings -> App.run settings cts.Token |> Async.AwaitTask | Error msg -> async { Log.error msg; return 1 } with | e -> Log.error $"Unhandled exception: {e}"; return 1 diff --git a/src/MinEdLauncher/Types.fs b/src/MinEdLauncher/Types.fs index 3423896..e703178 100644 --- a/src/MinEdLauncher/Types.fs +++ b/src/MinEdLauncher/Types.fs @@ -1,5 +1,6 @@ module MinEdLauncher.Types +open FSharp.Data open System open System.Diagnostics open Token @@ -77,6 +78,12 @@ type VersionInfo = SteamAware: bool Version: Version Mode: ProductMode } +type ProductMetadata = + { Hash: string + LocalFile: string + RemotePath: Uri + Size: int64 + Version: Version } type ProductDetails = { Sku: string Name: string @@ -88,12 +95,14 @@ type ProductDetails = Mode: ProductMode Directory: string GameArgs: string - ServerArgs: string } + ServerArgs: string + Metadata: ProductMetadata option } type MissingProductDetails = { Sku: string Name: string Filters: Set Directory: string } +type ProductManifest = XmlProvider<"""AppConfig.xmlb73379436461d1596b39f6aa07dd6d83724cca6d3366http://path.to/fileAudioConfiguration.xmlad79d0c6ca5988175b45c929ec039e86cd6967f32233http://path.to/file2"""> type Product = | Playable of ProductDetails | RequiresUpdate of ProductDetails diff --git a/tests/Product.fs b/tests/Product.fs index 5afe183..637e2df 100644 --- a/tests/Product.fs +++ b/tests/Product.fs @@ -25,7 +25,8 @@ open Expecto Mode = Offline Directory = "" GameArgs = "" - ServerArgs = "" } + ServerArgs = "" + Metadata = None } let getTimestamp = fun () -> (double)1 let hashFile = fun str -> Result.Ok Array.empty let token = EdSession.Empty