I’m running an i7 with 32GB of RAM, and I’ve reached 5800 rows/minute.
replication.py
from __future__ import annotations
import re
import subprocess
import time
import urllib.request
from datetime import datetime, timedelta, timezone
from pathlib import Path
COMPOSE_DIR = Path(r"C:\Users\Piotr\musicbrainz-docker")
LIVE_CHECK_SECONDS = 10 * 60
ROWS_PER_MINUTE = 5800
MNOZNIK = 12000
STATUS_SQL = """
SELECT count(*) FROM sir.pending_data;
""".strip()
def moja_procedura() -> None:
query = [
"docker", "compose", "exec", "-T", "db",
"psql", "-U", "musicbrainz", "-d", "musicbrainz_db",
"-t", "-A", "-c",
"SELECT current_replication_sequence "
"FROM replication_control WHERE id=1;"
]
result = subprocess.run(
query,
cwd=COMPOSE_DIR,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=True,
)
local_sequence = int(result.stdout.strip())
token_query = [
"docker", "compose", "exec", "-T", "musicbrainz",
"bash", "-c",
"carton exec -- perl -I/musicbrainz-server/lib -MDBDefs -e "
"'print DBDefs->REPLICATION_ACCESS_TOKEN'"
]
token_result = subprocess.run(
token_query,
cwd=COMPOSE_DIR,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=True,
)
token = token_result.stdout.strip()
url = (
"https://metabrainz.org/api/musicbrainz/"
f"replication-info?token={token}"
)
info = urllib.request.urlopen(url).read().decode("utf-8")
match = re.search(r'"last_packet"\s*:\s*"[^"]*?(\d+)', info)
if not match:
return
online_sequence = int(match.group(1))
packets = max(0, online_sequence - local_sequence)
minutes_needed = packets / 2 + MNOZNIK * packets / ROWS_PER_MINUTE
estimated_end = datetime.now().astimezone() + timedelta(minutes=minutes_needed)
print(f"Do pobrania: {packets}")
print(f"Szacowany czas zakończenia: {estimated_end.strftime('%A %H:%M')}")
def title(text: str) -> None:
print("=" * 72)
print(f"{text:^72}")
print("=" * 72)
def wait_for_key() -> None:
print()
print("=" * 72)
print(f"{'KONIEC':^72}")
print("=" * 72)
print()
print("Naciśnij dowolny klawisz, aby zakończyć...")
try:
import msvcrt
msvcrt.getch()
except Exception:
input()
def read_mirror_tail() -> list[str]:
"""Read recent replication log lines, including the rotated log."""
cmd = [
"docker", "compose", "exec", "-T", "musicbrainz",
"sh", "-c",
"for f in mirror.log mirror.log.1; do if [ -f \"$f\" ]; then tail -n 200 \"$f\"; fi; done",
]
r = subprocess.run(
cmd,
cwd=COMPOSE_DIR,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
if r.returncode != 0:
return []
return r.stdout.splitlines()
def packet_download_lines(lines: list[str]) -> list[str]:
"""Return timestamped lines that identify replication packet progress."""
result: list[str] = []
for line in lines:
has_time = bool(re.search(
r"\b\d{4}[-/]\d{2}[-/]\d{2}[^0-9]+\d{2}:\d{2}:\d{2}\b",
line,
))
if not has_time:
continue
has_packet_info = bool(re.search(
r"XID|Replication packet|replication[-_]?\d+|packet|download|"
r"downloading|fetch|fetching|pobier|pakiet",
line,
re.I,
))
if has_packet_info:
result.append(line)
return result
def packet_key(line: str) -> str:
"""Return a stable key for one replication packet, when possible."""
m = re.search(r"Replication packet #?(\d+)", line, re.I)
if m:
return "packet:" + m.group(1)
m = re.search(r"\bXID\s*[:=#]?\s*(\d+)", line, re.I)
if m:
return "xid:" + m.group(1)
m = re.search(r"replication[-_]?([0-9]+)", line, re.I)
if m:
return "replication:" + m.group(1)
return line
def start_replication() -> subprocess.Popen:
"""Start replication.sh; packet progress is read from mirror.log."""
cmd = [
"docker", "compose", "exec", "-T",
"musicbrainz", "replication.sh",
]
return subprocess.Popen(
cmd,
cwd=COMPOSE_DIR,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
)
def monitor_replication(process: subprocess.Popen, baseline: set[str]) -> int:
"""Poll mirror.log while replication.sh is running and show new packets."""
title("MUSICBRAINZ REPLICATION")
print()
moja_procedura()
print()
print("Pobieranie paczek (UTC / XID):")
print("-" * 72)
packet_count = 0
seen_lines: set[str] = set(baseline)
seen_packets: set[str] = set()
while process.poll() is None:
for line in read_mirror_tail():
for useful in packet_download_lines([line]):
if useful in seen_lines:
continue
seen_lines.add(useful)
key = packet_key(useful)
if key in seen_packets:
continue
seen_packets.add(key)
packet_count += 1
print(f"[{packet_count:>2}] {useful}", flush=True)
time.sleep(1)
# Catch the final packet and a possible mirror.log -> mirror.log.1 rotation.
for _ in range(3):
for line in read_mirror_tail():
for useful in packet_download_lines([line]):
if useful in seen_lines:
continue
seen_lines.add(useful)
key = packet_key(useful)
if key in seen_packets:
continue
seen_packets.add(key)
packet_count += 1
print(f"[{packet_count:>2}] {useful}", flush=True)
time.sleep(0.5)
rc = process.wait()
print("-" * 72)
print(f"Liczba pobranych paczek: {packet_count}")
return rc
def run_status_query() -> int:
cmd = [
"docker", "compose", "exec", "-T", "db",
"psql", "-U", "musicbrainz", "-d", "musicbrainz_db",
"-t", "-c", STATUS_SQL,
]
r = subprocess.run(
cmd,
cwd=COMPOSE_DIR,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
if r.returncode != 0:
raise RuntimeError(r.stderr.strip() or "psql failed")
return int(r.stdout.strip())
def live_indexing(initial_status: int | None = None) -> None:
title("MUSICBRAINZ LIVE INDEXING")
print()
check_no = 0
first_status = initial_status
while True:
check_no += 1
checked_at = datetime.now().astimezone()
if first_status is not None:
queued = first_status
first_status = None
else:
queued = run_status_query()
print(
f"#{check_no}\t{checked_at.strftime('%d.%m.%Y %H:%M:%S')}\t"
f"Queued: {queued}"
)
if queued == 0:
break
if queued < ROWS_PER_MINUTE * 10:
time.sleep(60)
else:
time.sleep(LIVE_CHECK_SECONDS)
def main() -> int:
if not COMPOSE_DIR.exists():
print(f"ERROR: brak katalogu {COMPOSE_DIR}")
return 1
baseline = set(read_mirror_tail())
process = start_replication()
rc = monitor_replication(process, baseline)
print()
if rc != 0:
print(f"replication.sh zakończył się kodem {rc}.")
return rc
title("REPLICATION COMPLETE")
try:
# First queue reading immediately after replication.
queued = run_status_query()
minutes_needed = queued / ROWS_PER_MINUTE
estimated_end = datetime.now().astimezone() + timedelta(minutes=minutes_needed)
print()
print(f"Queued: {queued}")
print(f"Szacowany czas zakończenia: {estimated_end.strftime('%A %H:%M')}")
print()
live_indexing(queued)
except KeyboardInterrupt:
print("\nPrzerwano przez użytkownika.")
return 130
except Exception as exc:
print(f"\nERROR: {exc}")
return 1
wait_for_key()
return 0
if __name__ == "__main__":
raise SystemExit(main())
indexer.ini
[database]
dbname = musicbrainz_db
host = ${MUSICBRAINZ_POSTGRES_SERVER}
password = ${POSTGRES_PASSWORD}
port = 5432
user = ${POSTGRES_USER}
[solr]
uri = http://${MUSICBRAINZ_SEARCH_SERVER}
batch_size = 400
[sir]
import_threads = 16
index_limit = 1000000
live_index_batch_size = 800
max_retries = 4
poll_interval = 5
process_delay = 15
query_batch_size = 5000
wscompat = on
[sentry]
dsn =
