using System.Text.Json; using System.Text.Json.Serialization; using Cronos; using Renci.SshNet; using Renci.SshNet.Sftp; using Serilog; var resetEvent = new ManualResetEvent(false); using var tokenSource = new CancellationTokenSource(); Console.CancelKeyPress += (sender, eventArgs) => { resetEvent?.Set(); tokenSource?.Cancel(); eventArgs.Cancel = true; Log.CloseAndFlush(); }; SetupLogging(); var config = LoadConfig(); Log.Information("Configuration file loaded. Beginning initial grab."); await RunJob(config, tokenSource.Token); Log.Information("Initial grab complete. Initializing schedule."); Task.Run(() => ScheduleJobs(config, tokenSource.Token)); resetEvent.WaitOne(); return; static void SetupLogging() { Log.Logger = new LoggerConfiguration() .MinimumLevel.Information() .WriteTo.Console() .WriteTo.File("log.txt", rollingInterval: RollingInterval.Day, rollOnFileSizeLimit: true, retainedFileCountLimit: 10) .CreateLogger(); } static Config LoadConfig() { var serializationOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, WriteIndented = true }; if (!File.Exists("/config.json")) { Log.Fatal("No config file found, creating dummy config."); CreateConfig(serializationOptions); Environment.Exit(1); } var configFile = File.ReadAllText("/config.json"); return JsonSerializer.Deserialize(configFile, serializationOptions); } static void CreateConfig(JsonSerializerOptions options) { var dummyGrab = new GrabTarget(@"/path/to/source", @"/path/to/destination"); var dummySFtp = new SFtpTarget("ftp.domain.example", 22, "username", false, "password"); var config = new Config(dummySFtp, [dummyGrab, dummyGrab], "* * * * *"); File.WriteAllText("/config.json", JsonSerializer.Serialize(config, options)); } static async Task ScheduleJobs(Config config, CancellationToken token) { var schedule = CronExpression.Parse(config.Schedule); while (!token.IsCancellationRequested) { var now = DateTimeOffset.Now; var nextJob = schedule.GetNextOccurrence(now, TimeZoneInfo.Local); if (!nextJob.HasValue) break; Log.Information($"Next scheduled scan at {nextJob.Value}"); var delay = nextJob.Value - now; if (delay > TimeSpan.Zero) { await Task.Delay(delay, token); } await RunJob(config, token); } } static async Task RunJob(Config config, CancellationToken token) { using var client = GetClient(config.FTP); var buffer = Math.Clamp(config.BufferSizeMB ?? 1, 0.5f, 4); client.BufferSize = (uint)(1024 * 1024 * buffer); Log.Information($"Connecting to {config.FTP.Host}:{config.FTP.Port} as {config.FTP.UserName}"); await client.ConnectAsync(token); foreach (var target in config.Targets) { Log.Information($"Beginning grab operation for '{target.Source}' to '{target.Destination}'"); await RecurseDirectory(client, target.Source, target.Destination, false, token); Log.Information($"Ending grab operation for '{target.Source}' to '{target.Destination}'"); } } static async Task RecurseDirectory(SftpClient client, string source, string destination, bool deleteDirectory, CancellationToken token) { Log.Information($"Scanning directory '{source}'"); await foreach (var item in client.ListDirectoryAsync(source, token)) { if (item.Name is ".." or ".") continue; if (item.IsDirectory) { var newPath = Path.Combine(destination, item.Name); await RecurseDirectory(client, item.FullName, newPath, true, token); continue; } await DownloadFile(client, item, destination, token); Log.Information($"Deleting '{item.Name}'"); await client.DeleteAsync(item.FullName, token); } if (deleteDirectory) { await client.DeleteDirectoryAsync(source, token); Log.Information($"Deleted directory '{source}'"); } } static async Task DownloadFile(SftpClient client, ISftpFile item, string destination, CancellationToken token) { Directory.CreateDirectory(destination); await using var stream = File.Open(Path.Combine(destination, item.Name), FileMode.Create, FileAccess.Write); Log.Information($"Downloading '{item.FullName}' to '{stream.Name}'"); await client.DownloadFileAsync(item.FullName, stream, token); } static SftpClient GetClient(SFtpTarget target) { return !target.usePrivateKey ? new SftpClient(target.Host, target.Port, target.UserName, target.Password) : new SftpClient(target.Host, target.Port, target.UserName, new PrivateKeyFile(target.KeyFile)); } public record Config(SFtpTarget FTP, GrabTarget[] Targets, string Schedule, float? BufferSizeMB = null); public record SFtpTarget(string Host, int Port, string UserName, bool usePrivateKey, string? Password = null, string? KeyFile = null); public record GrabTarget(string Source, string Destination);