Tool for replication and Live Indexing, Docker Desktop

I’m running an i7 with 32GB of RAM, and I’ve reached 5800 rows/minute.

replication.py

from __future__ import annotations

import re
import subprocess
import time
import urllib.request
from datetime import datetime, timedelta, timezone
from pathlib import Path

COMPOSE_DIR = Path(r"C:\Users\Piotr\musicbrainz-docker")

LIVE_CHECK_SECONDS = 10 * 60
ROWS_PER_MINUTE = 5800
MNOZNIK = 12000
STATUS_SQL = """
SELECT count(*) FROM sir.pending_data;
""".strip()


def moja_procedura() -> None:
    query = [
        "docker", "compose", "exec", "-T", "db",
        "psql", "-U", "musicbrainz", "-d", "musicbrainz_db",
        "-t", "-A", "-c",
        "SELECT current_replication_sequence "
        "FROM replication_control WHERE id=1;"
    ]

    result = subprocess.run(
        query,
        cwd=COMPOSE_DIR,
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        check=True,
    )

    local_sequence = int(result.stdout.strip())

    token_query = [
        "docker", "compose", "exec", "-T", "musicbrainz",
        "bash", "-c",
        "carton exec -- perl -I/musicbrainz-server/lib -MDBDefs -e "
        "'print DBDefs->REPLICATION_ACCESS_TOKEN'"
    ]

    token_result = subprocess.run(
        token_query,
        cwd=COMPOSE_DIR,
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        check=True,
    )

    token = token_result.stdout.strip()

    url = (
        "https://metabrainz.org/api/musicbrainz/"
        f"replication-info?token={token}"
    )

    info = urllib.request.urlopen(url).read().decode("utf-8")
    match = re.search(r'"last_packet"\s*:\s*"[^"]*?(\d+)', info)

    if not match:
        return

    online_sequence = int(match.group(1))
    packets = max(0, online_sequence - local_sequence)

    minutes_needed = packets / 2 + MNOZNIK * packets / ROWS_PER_MINUTE
    estimated_end = datetime.now().astimezone() + timedelta(minutes=minutes_needed)
    
    print(f"Do pobrania: {packets}")
    print(f"Szacowany czas zakończenia: {estimated_end.strftime('%A %H:%M')}")


def title(text: str) -> None:
    print("=" * 72)
    print(f"{text:^72}")
    print("=" * 72)


def wait_for_key() -> None:
    print()
    print("=" * 72)
    print(f"{'KONIEC':^72}")
    print("=" * 72)
    print()
    print("Naciśnij dowolny klawisz, aby zakończyć...")
    try:
        import msvcrt
        msvcrt.getch()
    except Exception:
        input()


def read_mirror_tail() -> list[str]:
    """Read recent replication log lines, including the rotated log."""
    cmd = [
        "docker", "compose", "exec", "-T", "musicbrainz",
        "sh", "-c",
        "for f in mirror.log mirror.log.1; do if [ -f \"$f\" ]; then tail -n 200 \"$f\"; fi; done",
    ]
    r = subprocess.run(
        cmd,
        cwd=COMPOSE_DIR,
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
    )
    if r.returncode != 0:
        return []
    return r.stdout.splitlines()


def packet_download_lines(lines: list[str]) -> list[str]:
    """Return timestamped lines that identify replication packet progress."""
    result: list[str] = []

    for line in lines:
        has_time = bool(re.search(
            r"\b\d{4}[-/]\d{2}[-/]\d{2}[^0-9]+\d{2}:\d{2}:\d{2}\b",
            line,
        ))
        if not has_time:
            continue

        has_packet_info = bool(re.search(
            r"XID|Replication packet|replication[-_]?\d+|packet|download|"
            r"downloading|fetch|fetching|pobier|pakiet",
            line,
            re.I,
        ))
        if has_packet_info:
            result.append(line)

    return result


def packet_key(line: str) -> str:
    """Return a stable key for one replication packet, when possible."""
    m = re.search(r"Replication packet #?(\d+)", line, re.I)
    if m:
        return "packet:" + m.group(1)

    m = re.search(r"\bXID\s*[:=#]?\s*(\d+)", line, re.I)
    if m:
        return "xid:" + m.group(1)

    m = re.search(r"replication[-_]?([0-9]+)", line, re.I)
    if m:
        return "replication:" + m.group(1)

    return line


def start_replication() -> subprocess.Popen:
    """Start replication.sh; packet progress is read from mirror.log."""
    cmd = [
        "docker", "compose", "exec", "-T",
        "musicbrainz", "replication.sh",
    ]

    return subprocess.Popen(
        cmd,
        cwd=COMPOSE_DIR,
        stdout=subprocess.DEVNULL,
        stderr=subprocess.STDOUT,
    )


def monitor_replication(process: subprocess.Popen, baseline: set[str]) -> int:
    """Poll mirror.log while replication.sh is running and show new packets."""
    title("MUSICBRAINZ REPLICATION")
    print()
    moja_procedura()
    print()
    print("Pobieranie paczek (UTC / XID):")
    print("-" * 72)

    packet_count = 0
    seen_lines: set[str] = set(baseline)
    seen_packets: set[str] = set()

    while process.poll() is None:
        for line in read_mirror_tail():
            for useful in packet_download_lines([line]):
                if useful in seen_lines:
                    continue
                seen_lines.add(useful)

                key = packet_key(useful)
                if key in seen_packets:
                    continue
                seen_packets.add(key)
                packet_count += 1
                print(f"[{packet_count:>2}] {useful}", flush=True)

        time.sleep(1)

    # Catch the final packet and a possible mirror.log -> mirror.log.1 rotation.
    for _ in range(3):
        for line in read_mirror_tail():
            for useful in packet_download_lines([line]):
                if useful in seen_lines:
                    continue
                seen_lines.add(useful)

                key = packet_key(useful)
                if key in seen_packets:
                    continue
                seen_packets.add(key)
                packet_count += 1
                print(f"[{packet_count:>2}] {useful}", flush=True)
        time.sleep(0.5)

    rc = process.wait()

    print("-" * 72)
    print(f"Liczba pobranych paczek: {packet_count}")
    return rc


def run_status_query() -> int:
    cmd = [
        "docker", "compose", "exec", "-T", "db",
        "psql", "-U", "musicbrainz", "-d", "musicbrainz_db",
        "-t", "-c", STATUS_SQL,
    ]

    r = subprocess.run(
        cmd,
        cwd=COMPOSE_DIR,
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
    )

    if r.returncode != 0:
        raise RuntimeError(r.stderr.strip() or "psql failed")

    return int(r.stdout.strip())


def live_indexing(initial_status: int | None = None) -> None:
    title("MUSICBRAINZ LIVE INDEXING")
    print()

    check_no = 0
    first_status = initial_status

    while True:
        check_no += 1
        checked_at = datetime.now().astimezone()

        if first_status is not None:
            queued = first_status
            first_status = None
        else:
            queued = run_status_query()

        print(
            f"#{check_no}\t{checked_at.strftime('%d.%m.%Y %H:%M:%S')}\t"
            f"Queued: {queued}"
        )

        if queued == 0:
            break

        if queued < ROWS_PER_MINUTE * 10:
            time.sleep(60)
        else:
            time.sleep(LIVE_CHECK_SECONDS)

def main() -> int:
    if not COMPOSE_DIR.exists():
        print(f"ERROR: brak katalogu {COMPOSE_DIR}")
        return 1

    baseline = set(read_mirror_tail())
    process = start_replication()
    rc = monitor_replication(process, baseline)

    print()
    if rc != 0:
        print(f"replication.sh zakończył się kodem {rc}.")
        return rc

    title("REPLICATION COMPLETE")

    try:
        # First queue reading immediately after replication.
        queued = run_status_query()
        minutes_needed = queued / ROWS_PER_MINUTE
        estimated_end = datetime.now().astimezone() + timedelta(minutes=minutes_needed)

        print()
        print(f"Queued: {queued}")
        print(f"Szacowany czas zakończenia: {estimated_end.strftime('%A %H:%M')}")
        print()

        live_indexing(queued)
    except KeyboardInterrupt:
        print("\nPrzerwano przez użytkownika.")
        return 130
    except Exception as exc:
        print(f"\nERROR: {exc}")
        return 1

    wait_for_key()
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

indexer.ini

[database]
dbname = musicbrainz_db
host = ${MUSICBRAINZ_POSTGRES_SERVER}
password = ${POSTGRES_PASSWORD}
port = 5432
user = ${POSTGRES_USER}

[solr]
uri = http://${MUSICBRAINZ_SEARCH_SERVER}
batch_size = 400

[sir]
import_threads = 16
index_limit = 1000000
live_index_batch_size = 800
max_retries = 4
poll_interval = 5
process_delay = 15
query_batch_size = 5000
wscompat = on

[sentry]
dsn =