fpc-msgbase/tests/tools/make_hudson_sample.pas

{
  make_hudson_sample.pas - build a populated multi-conference
  Hudson base for testing by copying real messages from JAM.

  Output: /tmp/ma_hudson_sample/  (move/back up elsewhere manually)

  Layout: 20 JAM areas found under ~/fidonet/msg/jam/ map to
  Hudson boards 1..20.  Up to 50 messages per board.
}

program make_hudson_sample;

{$mode objfpc}{$H+}

uses
  SysUtils, Classes,
  ma.types, ma.events, ma.api,
  ma.fmt.jam, ma.fmt.jam.uni,
  ma.fmt.hudson, ma.fmt.hudson.uni;

const
  JAM_DIR       = '/home/ken/fidonet/msg/jam';
  HUDSON_DIR    = '/tmp/ma_hudson_sample';
  MAX_BOARDS    = 20;

  { Hudson's MSGIDX record stores MsgNum as a signed 16-bit value
    (smallint), so the absolute ceiling is 32767 messages per
    base.  Stop well short of that to leave room for future
    writes during tests. }
  HUDSON_HARD_LIMIT = 30000;
  MSGS_PER_AREA     = HUDSON_HARD_LIMIT div MAX_BOARDS;   { 1500 }

function ListJamAreas: TStringList;
var
  sr: TSearchRec;
  stem: string;
begin
  Result := TStringList.Create;
  Result.Sorted := True;
  Result.Duplicates := dupIgnore;
  if FindFirst(JAM_DIR + '/*.jhr', faAnyFile, sr) = 0 then
  try
    repeat
      stem := ChangeFileExt(sr.Name, '');
      Result.Add(stem);
    until FindNext(sr) <> 0;
  finally
    FindClose(sr);
  end;
end;

function FileSizeOrZero(const APath: string): int64;
var
  fh: file of byte;
begin
  Result := 0;
  if not FileExists(APath) then exit;
  AssignFile(fh, APath);
  {$I-} Reset(fh); {$I+}
  if IOResult <> 0 then exit;
  Result := FileSize(fh);
  Close(fh);
end;

var
  allAreas:  TStringList;
  picked:    TStringList;
  manifest:  TStringList;
  hudson:    TMessageBase;
  jam:       TMessageBase;
  msg:       TUniMessage;
  board:     word;
  i, n, copied, totalCopied: longint;
  areaName:  string;
begin
  WriteLn('make_hudson_sample: building multi-board Hudson test base');
  WriteLn;
  WriteLn('  source areas: ', JAM_DIR);
  WriteLn('  output base:  ', HUDSON_DIR);
  WriteLn;

  if DirectoryExists(HUDSON_DIR) then begin
    WriteLn('Removing existing ', HUDSON_DIR);
    ExecuteProcess('/bin/sh', ['-c',
                   SysUtils.Format('rm -rf %s', [HUDSON_DIR])]);
  end;

  allAreas := ListJamAreas;
  try
    WriteLn('Found ', allAreas.Count, ' JAM areas under ', JAM_DIR);

    { Skip areas smaller than a header (probably empty/stub) and
      cap at MAX_BOARDS. }
    picked := TStringList.Create;
    try
      i := 0;
      while (picked.Count < MAX_BOARDS) and (i < allAreas.Count) do begin
        areaName := allAreas[i];
        if FileSizeOrZero(JAM_DIR + '/' + areaName + '.jhr') > 1024 then
          picked.Add(areaName);
        Inc(i);
      end;
      WriteLn('Selected ', picked.Count, ' areas for boards 1..',
              picked.Count);
      WriteLn;

      manifest := TStringList.Create;
      try
        manifest.Add('# Hudson test-base conference manifest');
        manifest.Add('# generated by make_hudson_sample.pas');
        manifest.Add('# format: board <tab> area <tab> messages_copied');
        manifest.Add('');

        hudson := MessageBaseOpen(mbfHudson, HUDSON_DIR, momCreate);
        try
          if not hudson.Open then begin
            WriteLn('error: cannot create Hudson base at ', HUDSON_DIR);
            Halt(1);
          end;

          totalCopied := 0;
          for board := 1 to picked.Count do begin
            areaName := picked[board - 1];
            Write(SysUtils.Format('  board %2d  %-20s ',
                                  [board, areaName]));
            jam := MessageBaseOpen(mbfJam,
                                   JAM_DIR + '/' + areaName,
                                   momReadOnly);
            try
              if not jam.Open then begin
                WriteLn('(skipped, open failed)');
                manifest.Add(SysUtils.Format('%d'#9'%s'#9'0 (open failed)',
                                            [board, areaName]));
                Continue;
              end;
              n := jam.MessageCount;
              copied := 0;
              i := 0;
              while i < n do begin
                if copied >= MSGS_PER_AREA then break;
                if totalCopied + copied >= HUDSON_HARD_LIMIT then break;
                if jam.ReadMessage(i, msg) then begin
                  msg.Board   := board;
                  msg.AreaTag := areaName;
                  hudson.WriteMessage(msg);
                  Inc(copied);
                end;
                Inc(i);
              end;
              WriteLn('copied ', copied, ' of ', n, ' messages');
              manifest.Add(SysUtils.Format('%d'#9'%s'#9'%d',
                                          [board, areaName, copied]));
              Inc(totalCopied, copied);
            finally
              jam.Close;
              jam.Free;
            end;
          end;

          WriteLn;
          WriteLn('Total messages in Hudson base: ', hudson.MessageCount);
          WriteLn('Total copied this run:         ', totalCopied);
        finally
          hudson.Close;
          hudson.Free;
        end;

        manifest.Add('');
        manifest.Add(SysUtils.Format('# total_messages %d', [totalCopied]));
        manifest.SaveToFile(HUDSON_DIR + '/CONFERENCES.TXT');
      finally
        manifest.Free;
      end;
    finally
      picked.Free;
    end;
  finally
    allAreas.Free;
  end;

  WriteLn;
  WriteLn('Done.  Hudson base is at ', HUDSON_DIR);
  WriteLn('Files:');
  ExecuteProcess('/bin/sh', ['-c',
                 SysUtils.Format('ls -la %s/', [HUDSON_DIR])]);
end.