using System.Text.Json; using System.Text.Json.Serialization; using Cronos; using Renci.SshNet; using Renci.SshNet.Sftp; using Serilog; using Serilog.Core; using Serilog.Events; var resetEvent = new ManualResetEvent(false); using var tokenSource = new CancellationTokenSource(); Console.CancelKeyPress += (_, eventArgs) => { Log.Debug("Canceling with CancelKeyPress"); Dispose(); eventArgs.Cancel = true; Log.CloseAndFlush(); }; AppDomain.CurrentDomain.ProcessExit += (_, _) => Dispose(); SetupLogging(); var config = LoadConfig(); Log.Information("Configuration file loaded. Beginning initial grab."); await RunJob(config, tokenSource.Token); Log.Information("Initial grab complete. Initializing schedule."); Task.Run(() => ScheduleJobs(config, tokenSource.Token)); resetEvent.WaitOne(); return; void Dispose() { resetEvent?.Set(); tokenSource?.Cancel(); Log.CloseAndFlush(); } static void SetupLogging() { var minLevel = (Environment.GetEnvironmentVariable("LOG_LEVEL") ?? "info").ToUpperInvariant() switch { "VERBOSE" => LogEventLevel.Verbose, "DEBUG" => LogEventLevel.Debug, "WARNING" => LogEventLevel.Warning, "ERROR" => LogEventLevel.Error, "FATAL" => LogEventLevel.Fatal, _ => LogEventLevel.Information }; var levelSwitch = new LoggingLevelSwitch(minLevel); Log.Logger = new LoggerConfiguration() .MinimumLevel.ControlledBy(levelSwitch) .WriteTo.Console() .WriteTo.File("log.txt", rollingInterval: RollingInterval.Day, rollOnFileSizeLimit: true, retainedFileCountLimit: 10) .CreateLogger(); Log.Debug($"Logging initialized at {minLevel}"); } static Config LoadConfig() { var serializationOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, WriteIndented = true }; if (!File.Exists("/config.json")) { Log.Fatal("No config file found, creating dummy config."); CreateConfig(serializationOptions); Environment.Exit(1); } var configFile = File.ReadAllText("/config.json"); return JsonSerializer.Deserialize(configFile, serializationOptions); } static void CreateConfig(JsonSerializerOptions options) { var dummyGrab = new GrabTarget(@"/path/to/source", @"/path/to/destination"); var dummySFtp = new SFtpTarget("ftp.domain.example", 22, "username", false, "password"); var config = new Config(dummySFtp, [dummyGrab, dummyGrab], "* * * * *"); File.WriteAllText("/config.json", JsonSerializer.Serialize(config, options)); } static async Task ScheduleJobs(Config config, CancellationToken token) { var schedule = CronExpression.Parse(config.Schedule); while (!token.IsCancellationRequested) { var now = DateTimeOffset.Now; var nextJob = schedule.GetNextOccurrence(now, TimeZoneInfo.Local); if (!nextJob.HasValue) break; Log.Information($"Next scheduled scan at {nextJob.Value}"); var delay = nextJob.Value - now; if (delay > TimeSpan.Zero) { await Task.Delay(delay, token); } await RunJob(config, token); } } static async Task RunJob(Config config, CancellationToken token) { using var client = GetClient(config.FTP); Log.Information($"Connecting to {config.FTP.Host}:{config.FTP.Port} as {config.FTP.UserName}"); var buffer = Math.Clamp(config.BufferSizeMB ?? 1, 0.5f, 4); client.BufferSize = (uint)(1024 * 1024 * buffer); Log.Debug($"Connection buffer size: {client.BufferSize}"); await client.ConnectAsync(token); foreach (var target in config.Targets) { Log.Information($"Beginning grab operation for '{target.Source}' to '{target.Destination}'"); await RecurseDirectory(client, target.Source, target.Destination, false, token); Log.Information($"Ending grab operation for '{target.Source}' to '{target.Destination}'"); } } static async Task RecurseDirectory(SftpClient client, string source, string destination, bool deleteDirectory, CancellationToken token) { Log.Information($"Scanning directory '{source}'"); await Parallel.ForEachAsync(client.ListDirectoryAsync(source, token), token, async (item,token) => { if (item.Name is ".." or ".") return; if (item.IsDirectory) { var newPath = Path.Combine(destination, item.Name); await RecurseDirectory(client, item.FullName, newPath, true, token); return; } await DownloadFile(client, item, destination, token); Log.Information($"Deleting '{item.Name}'"); await client.DeleteAsync(item.FullName, token); }); if (deleteDirectory) { await client.DeleteDirectoryAsync(source, token); Log.Information($"Deleted directory '{source}'"); } } static async Task DownloadFile(SftpClient client, ISftpFile item, string destination, CancellationToken token) { Directory.CreateDirectory(destination); await using var stream = File.Open(Path.Combine(destination, item.Name), FileMode.Create, FileAccess.Write); Log.Information($"Downloading '{item.FullName}' to '{stream.Name}'"); await client.DownloadFileAsync(item.FullName, stream, token); } static SftpClient GetClient(SFtpTarget target) { return !target.usePrivateKey ? new SftpClient(target.Host, target.Port, target.UserName, target.Password) : new SftpClient(target.Host, target.Port, target.UserName, new PrivateKeyFile(target.KeyFile)); } public record Config(SFtpTarget FTP, GrabTarget[] Targets, string Schedule, float? BufferSizeMB = null); public record SFtpTarget(string Host, int Port, string UserName, bool usePrivateKey, string? Password = null, string? KeyFile = null); public record GrabTarget(string Source, string Destination);