FtpThing/Program.cs

using System.Text.Json;
using System.Text.Json.Serialization;
using Cronos;
using Renci.SshNet;
using Renci.SshNet.Sftp;
using Serilog;
using Serilog.Core;
using Serilog.Events;

var resetEvent = new ManualResetEvent(false);
using var tokenSource = new CancellationTokenSource();

Console.CancelKeyPress += (_, eventArgs) =>
{
    Log.Debug("Canceling with CancelKeyPress");
    Dispose();
    eventArgs.Cancel = true;
    Log.CloseAndFlush();
};

AppDomain.CurrentDomain.ProcessExit += (_, _) => Dispose();

SetupLogging();

var config = LoadConfig();
Log.Information("Configuration file loaded. Beginning initial grab.");
await RunJob(config, tokenSource.Token);
Log.Information("Initial grab complete. Initializing schedule.");
Task.Run(() => ScheduleJobs(config, tokenSource.Token));

resetEvent.WaitOne();

return;

void Dispose()
{
    resetEvent?.Set();
    tokenSource?.Cancel();
    Log.CloseAndFlush();
}

static void SetupLogging()
{
    var minLevel = (Environment.GetEnvironmentVariable("LOG_LEVEL") ?? "info").ToUpperInvariant() switch
    {
        "VERBOSE" => LogEventLevel.Verbose,
        "DEBUG" => LogEventLevel.Debug,
        "WARNING" => LogEventLevel.Warning,
        "ERROR" => LogEventLevel.Error,
        "FATAL" => LogEventLevel.Fatal,
        _ => LogEventLevel.Information
    };
    var levelSwitch =  new LoggingLevelSwitch(minLevel);
    
    Log.Logger = new LoggerConfiguration()
        .MinimumLevel.ControlledBy(levelSwitch)
        .WriteTo.Console()
        .WriteTo.File("log.txt",
            rollingInterval: RollingInterval.Day, 
            rollOnFileSizeLimit: true,
            retainedFileCountLimit: 10)
        .CreateLogger();
    
    Log.Debug($"Logging initialized at {minLevel}");
}

static Config LoadConfig()
{
    var serializationOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
    {
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault,
        WriteIndented = true
    };

    if (!File.Exists("/config.json"))
    {
        Log.Fatal("No config file found, creating dummy config.");
        CreateConfig(serializationOptions);
        Environment.Exit(1);
    }

    var configFile = File.ReadAllText("/config.json");
    return JsonSerializer.Deserialize<Config>(configFile, serializationOptions);
}

static void CreateConfig(JsonSerializerOptions options)
{
    var dummyGrab = new GrabTarget(@"/path/to/source", @"/path/to/destination");
    var dummySFtp = new SFtpTarget("ftp.domain.example", 22, "username", false, "password");
    var config = new Config(dummySFtp, [dummyGrab, dummyGrab], "* * * * *");
    File.WriteAllText("/config.json", JsonSerializer.Serialize(config, options));
}

static async Task ScheduleJobs(Config config, CancellationToken token)
{
    var schedule = CronExpression.Parse(config.Schedule);
    
    while (!token.IsCancellationRequested)
    {
        var now = DateTimeOffset.Now;
        var nextJob = schedule.GetNextOccurrence(now, TimeZoneInfo.Local);
        
        if (!nextJob.HasValue) break;
        
        Log.Information($"Next scheduled scan at {nextJob.Value}");
        var delay = nextJob.Value - now;
        if (delay > TimeSpan.Zero)
        {
            await Task.Delay(delay, token);
        }
        
        await RunJob(config, token);
    }
}

static async Task RunJob(Config config, CancellationToken token)
{
    using var client = GetClient(config.FTP);
    Log.Information($"Connecting to {config.FTP.Host}:{config.FTP.Port} as {config.FTP.UserName}");
    
    var buffer = Math.Clamp(config.BufferSizeMB ?? 1, 0.5f, 4);
    client.BufferSize = (uint)(1024 * 1024 * buffer);
    Log.Debug($"Connection buffer size: {client.BufferSize}");
    
    await client.ConnectAsync(token);
    foreach (var target in config.Targets)
    {
        Log.Information($"Beginning grab operation for '{target.Source}' to '{target.Destination}'");
        await RecurseDirectory(client, target.Source, target.Destination, false, token);
        Log.Information($"Ending grab operation for '{target.Source}' to '{target.Destination}'");
    }
}

static async Task RecurseDirectory(SftpClient client, string source, string destination, bool deleteDirectory, CancellationToken token)
{
    Log.Information($"Scanning directory '{source}'");
    await Parallel.ForEachAsync(client.ListDirectoryAsync(source, token), token, async (item,token) =>
    {
        if (item.Name is ".." or ".") return;

        if (item.IsDirectory)
        {
            var newPath = Path.Combine(destination, item.Name);
            await RecurseDirectory(client, item.FullName, newPath, true, token);
            return;
        }

        await DownloadFile(client, item, destination, token);
        Log.Information($"Deleting '{item.Name}'");
        await client.DeleteAsync(item.FullName, token);
    });

    if (deleteDirectory)
    {
        await client.DeleteDirectoryAsync(source, token);
        Log.Information($"Deleted directory '{source}'");
    }
}

static async Task DownloadFile(SftpClient client, ISftpFile item, string destination, CancellationToken token)
{
    Directory.CreateDirectory(destination);
    await using var stream = File.Open(Path.Combine(destination, item.Name), FileMode.Create, FileAccess.Write);
    Log.Information($"Downloading '{item.FullName}' to '{stream.Name}'");
    await client.DownloadFileAsync(item.FullName, stream, token);
}

static SftpClient GetClient(SFtpTarget target)
{
    return !target.usePrivateKey ? 
        new SftpClient(target.Host, target.Port, target.UserName, target.Password) 
      : new SftpClient(target.Host, target.Port, target.UserName, new PrivateKeyFile(target.KeyFile));
}


public record Config(SFtpTarget FTP, GrabTarget[] Targets, string Schedule, float? BufferSizeMB = null);
public record SFtpTarget(string Host, int Port, string UserName, bool usePrivateKey, string? Password = null, string? KeyFile = null);
public record GrabTarget(string Source, string Destination);