diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ad6dbed --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,65 @@ +name: CI + +on: + push: + branches: ["**"] + tags: + - "v*" + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Tests (unit + e2e) + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Upgrade pip + run: python -m pip install -U pip + + - name: Install project (editable) + run: python -m pip install -e . + + - name: Run tests + run: make test + + tag-stable: + name: Tag stable on version tag + runs-on: ubuntu-latest + needs: [test] + if: startsWith(github.ref, 'refs/tags/v') + + permissions: + contents: write + + steps: + - name: Checkout (full history for tags) + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Configure git user + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Force-update stable tag to this commit + env: + SHA: ${{ github.sha }} + run: | + git tag -f stable "${SHA}" + git push -f origin stable diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 1a8de72..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: CI - -on: - push: - branches: [ "**" ] - pull_request: - branches: [ "**" ] - -jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12"] - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Show Python version - run: python -V - - - name: Make main.py executable (optional) - run: chmod +x main.py || true - - - name: Install test dependencies (if any) - run: | - if [ -f requirements.txt ]; then - python -m pip install --upgrade pip - pip install -r requirements.txt - fi - - - name: Run tests - run: make test diff --git a/MIRRORS b/MIRRORS new file mode 100644 index 0000000..e0e3a2e --- /dev/null +++ b/MIRRORS @@ -0,0 +1,4 @@ +git@github.com:kevinveenbirkenbach/cleanup-failed-backups.git +ssh://git@git.veen.world:2201/kevinveenbirkenbach/cleanback.git +ssh://git@code.infinito.nexus:2201/kevinveenbirkenbach/cleanback.git +https://pypi.org/project/cleanback/ diff --git a/Makefile b/Makefile index 46f9c83..d18f7b1 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,17 @@ # Makefile for Cleanup Failed Backups -.PHONY: test install help +.PHONY: install help test test-unit test-e2e help: @echo "Available targets:" @echo " make test - Run unit tests" - @echo " make install - Show installation instructions" -test: +test: test-unit test-e2e + +test-unit: @echo ">> Running tests" - @python3 -m unittest -v test.py + @python3 -m unittest -v tests/unit/test_main.py -install: - @echo ">> Installation instructions:" - @echo " This software can be installed with pkgmgr:" - @echo " pkgmgr install cleanback" - @echo " See project: https://github.com/kevinveenbirkenbach/package-manager" +test-e2e: + docker build -f tests/e2e/Dockerfile.e2e -t cleanback-e2e . + docker run --rm cleanback-e2e diff --git a/README.md b/README.md index 54f8353..13ac92b 100644 --- a/README.md +++ b/README.md @@ -7,95 +7,134 @@ **Repository:** https://github.com/kevinveenbirkenbach/cleanup-failed-backups -This tool validates and (optionally) cleans up **failed Docker backup directories**. -It scans backup folders under `/Backups`, uses [`dirval`](https://github.com/kevinveenbirkenbach/directory-validator) to validate each subdirectory, and lets you delete the ones that fail validation. Validation runs **in parallel** for performance; deletions are controlled and can be interactive or automatic. +`cleanback` validates and (optionally) cleans up **failed Docker backup directories**. +It scans backup folders under `/Backups`, uses :contentReference[oaicite:0]{index=0} to validate each subdirectory, and lets you delete the ones that fail validation. + +Validation runs **in parallel** for performance; deletions are controlled and can be **interactive** or **fully automatic**. --- ## ✨ Highlights - **Parallel validation** of backup subdirectories -- Uses **`dirval`** (`directory-validator`) via CLI for robust validation +- Uses **`dirval`** (directory-validator) via CLI - **Interactive** or **non-interactive** deletion flow (`--yes`) - Supports validating a single backup **ID** or **all** backups +- Clean **Python package** with `pyproject.toml` +- **Unit + Docker-based E2E tests** --- ## πŸ“¦ Installation -This project is installable via **pkgmgr** (Kevin’s package manager). - -**New pkgmgr alias:** `cleanback` +### Via pip (recommended) ```bash -# Install pkgmgr first (if you don't have it): -# https://github.com/kevinveenbirkenbach/package-manager - -pkgmgr install cleanback +pip install cleanback ```` -> `dirval` is declared as a dependency (see `requirements.yml`) and will be resolved by pkgmgr. +This installs: + +* the `cleanback` CLI +* `dirval` as a dependency (declared in `pyproject.toml`) + +### Editable install (for development) + +```bash +git clone https://github.com/kevinveenbirkenbach/cleanup-failed-backups +cd cleanup-failed-backups +pip install -e . +``` --- ## πŸ”§ Requirements -* Python 3.8+ -* `dirval` available on PATH (resolved automatically by `pkgmgr install cleanback`) -* Access to `/Backups` directory tree +* Python **3.8+** +* Access to the `/Backups` directory tree +* `dirval` (installed automatically via pip dependency) --- ## πŸš€ Usage -The executable is `main.py`: +### CLI entrypoint + +After installation, the command is: ```bash -# Validate a single backup ID (under /Backups//backup-docker-to-local) -python3 main.py --id - -# Validate ALL backup IDs under /Backups/*/backup-docker-to-local -python3 main.py --all +cleanback ``` +### Validate a single backup ID + +```bash +cleanback --id +``` + +Validates directories under: + +``` +/Backups//backup-docker-to-local/* +``` + +### Validate all backups + +```bash +cleanback --all +``` + +Scans: + +``` +/Backups/*/backup-docker-to-local/* +``` + +--- + ### Common options -* `--dirval-cmd ` β€” command to run `dirval` (default: `dirval`) -* `--workers ` β€” parallel workers (default: CPU count, min 2) -* `--timeout ` β€” per-directory validation timeout (float supported; default: 300.0) -* `--yes` β€” **non-interactive**: auto-delete directories that fail validation +| Option | Description | +| -------------------- | ------------------------------------------------------------------ | +| `--dirval-cmd ` | Path or name of `dirval` executable (default: `dirval`) | +| `--workers ` | Parallel workers (default: CPU count, min 2) | +| `--timeout ` | Per-directory validation timeout (float supported, default: 300.0) | +| `--yes` | Non-interactive mode: delete failures automatically | + +--- ### Examples ```bash -# Validate a single backup and prompt for deletions on failures -python3 main.py --id 2024-09-01T12-00-00 +# Validate a single backup and prompt on failures +cleanback --id 2024-09-01T12-00-00 # Validate everything with 8 workers and auto-delete failures -python3 main.py --all --workers 8 --yes +cleanback --all --workers 8 --yes -# Use a custom dirval binary and shorter timeout -python3 main.py --all --dirval-cmd /usr/local/bin/dirval --timeout 5.0 +# Use a custom dirval binary and short timeout +cleanback --all --dirval-cmd /usr/local/bin/dirval --timeout 5.0 ``` --- ## πŸ§ͺ Tests +### Run all tests + ```bash make test ``` -This runs the unit tests in `test.py`. Tests create a temporary `/Backups`-like tree and a fake `dirval` to simulate success/failure/timeout behavior. - --- -## πŸ“ Project Layout +## πŸ”’ Safety & Design Notes -* `main.py` β€” CLI entry point (parallel validator + cleanup) -* `test.py` β€” unit tests -* `requirements.yml` β€” `pkgmgr` dependencies (includes `dirval`) -* `Makefile` β€” `make test` and an informational `make install` +* **No host filesystem is modified** during tests + (E2E tests run exclusively inside Docker) +* Deletions are **explicitly confirmed** unless `--yes` is used +* Timeouts are treated as **validation failures** +* Validation and deletion phases are **clearly separated** --- diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5c33f82 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "cleanback" +version = "0.1.0" +description = "Cleanup Failed Docker Backups β€” parallel validator (using dirval)" +readme = "README.md" +requires-python = ">=3.8" +license = { file = "LICENSE" } +authors = [{ name = "Kevin Veen-Birkenbach", email = "kevin@veen.world" }] +keywords = ["backup", "docker", "validation", "cleanup", "dirval"] +dependencies = [ + "dirval>=0.1.0", +] + +[project.urls] +Homepage = "https://github.com/kevinveenbirkenbach/cleanup-failed-backups" +Repository = "https://github.com/kevinveenbirkenbach/cleanup-failed-backups" + +[project.scripts] +cleanback = "cleanback.__main__:main" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/requirements.yml b/requirements.yml deleted file mode 100644 index 5d20c8b..0000000 --- a/requirements.yml +++ /dev/null @@ -1,2 +0,0 @@ -pkgmgr: - - dirval \ No newline at end of file diff --git a/src/cleanback/__init__.py b/src/cleanback/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/main.py b/src/cleanback/__main__.py similarity index 100% rename from main.py rename to src/cleanback/__main__.py diff --git a/tests/e2e/Dockerfile.e2e b/tests/e2e/Dockerfile.e2e new file mode 100644 index 0000000..079d65d --- /dev/null +++ b/tests/e2e/Dockerfile.e2e @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR /opt/app + +# Copy project +COPY . . + +# Install the project (editable is fine for tests) +RUN python -m pip install -U pip \ + && python -m pip install -e . \ + && python -m pip install -U unittest-xml-reporting >/dev/null 2>&1 || true + +# Create /Backups in container (our tests will use it) +RUN mkdir -p /Backups + +# Run E2E unittest +CMD ["python", "-m", "unittest", "-v", "tests.e2e.test_e2e_docker"] diff --git a/tests/e2e/test_e2e_docker.py b/tests/e2e/test_e2e_docker.py new file mode 100644 index 0000000..1efb770 --- /dev/null +++ b/tests/e2e/test_e2e_docker.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +FAKE_TIMEOUT_SLEEP = 0.3 +SHORT_TIMEOUT = "0.1" + + +FAKE_DIRVAL = f"""#!/usr/bin/env python3 +import sys, time, argparse, pathlib + +def main(): + p = argparse.ArgumentParser() + p.add_argument("path") + p.add_argument("--validate", action="store_true") + args = p.parse_args() + + d = pathlib.Path(args.path) + name = d.name.lower() + + if "timeout" in name: + time.sleep({FAKE_TIMEOUT_SLEEP}) + print("Simulated long run...") + return 0 + + if (d / "VALID").exists(): + print("ok") + return 0 + + print("failed") + return 1 + +if __name__ == "__main__": + sys.exit(main()) +""" + + +class CleanbackE2EDockerTests(unittest.TestCase): + """ + E2E test that uses real directories, but runs inside a Docker container. + It creates /Backups structure inside the container and invokes the app + via `python -m cleanback`. + """ + + def setUp(self): + # Create a real /Backups root inside the container + # (safe because we are in Docker) + self.backups_root = Path("/Backups") + self.backups_root.mkdir(parents=True, exist_ok=True) + + # Use a unique run folder so repeated runs don't collide + self.run_root = self.backups_root / f"E2E-{os.getpid()}" + self.run_root.mkdir(parents=True, exist_ok=True) + + # Create fake `dirval` executable on disk (real file, real chmod) + self.bin_dir = Path(tempfile.mkdtemp(prefix="cleanback-bin-")) + self.dirval = self.bin_dir / "dirval" + self.dirval.write_text(FAKE_DIRVAL, encoding="utf-8") + self.dirval.chmod(0o755) + + # Create real backup directory structure + # /Backups//backup-docker-to-local/{good,bad,timeout} + self.backup_id = "ID-E2E" + self.base = self.run_root / self.backup_id / "backup-docker-to-local" + self.base.mkdir(parents=True, exist_ok=True) + + self.good = self.base / "good" + self.bad = self.base / "bad" + self.timeout = self.base / "timeout" + for p in (self.good, self.bad, self.timeout): + p.mkdir(parents=True, exist_ok=True) + + (self.good / "VALID").write_text("1", encoding="utf-8") + + def tearDown(self): + # Cleanup what we created inside /Backups + # Keep it simple and robust (don't fail teardown) + try: + if self.run_root.exists(): + for p in sorted(self.run_root.rglob("*"), reverse=True): + try: + if p.is_dir(): + p.rmdir() + else: + p.unlink() + except Exception: + pass + try: + self.run_root.rmdir() + except Exception: + pass + except Exception: + pass + + try: + # Remove temp bin dir + if self.bin_dir.exists(): + for p in sorted(self.bin_dir.rglob("*"), reverse=True): + try: + if p.is_dir(): + p.rmdir() + else: + p.unlink() + except Exception: + pass + try: + self.bin_dir.rmdir() + except Exception: + pass + except Exception: + pass + + def test_e2e_id_mode_yes_deletes_failures(self): + env = os.environ.copy() + + # Prepend fake dirval path for this test run + env["PATH"] = f"{self.bin_dir}:{env.get('PATH','')}" + + # Run: python -m cleanback --id --yes + # We must point BACKUPS_ROOT to our run_root. Easiest: set /Backups = run_root + # But code currently has BACKUPS_ROOT = /Backups constant. + # + # Therefore, we create our test tree under /Backups (done above) and pass --id + # relative to that structure by using run_root/. To do that, we make + # run_root the direct child under /Backups, then we pass the composite id: + # "/". + composite_id = f"{self.run_root.name}/{self.backup_id}" + + cmd = [ + "python", "-m", "cleanback", + "--id", composite_id, + "--dirval-cmd", "dirval", + "--workers", "4", + "--timeout", SHORT_TIMEOUT, + "--yes", + ] + proc = subprocess.run(cmd, text=True, capture_output=True, env=env) + + self.assertEqual(proc.returncode, 0, msg=proc.stderr or proc.stdout) + self.assertTrue(self.good.exists(), "good should remain") + self.assertFalse(self.bad.exists(), "bad should be deleted") + self.assertFalse(self.timeout.exists(), "timeout should be deleted (timeout treated as failure)") + self.assertIn("Summary:", proc.stdout) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test.py b/tests/unit/test_main.py similarity index 97% rename from test.py rename to tests/unit/test_main.py index 393cee8..a43e511 100644 --- a/test.py +++ b/tests/unit/test_main.py @@ -8,10 +8,10 @@ import contextlib from pathlib import Path from unittest.mock import patch -# Import cleanup main.py -HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE)) -import main # noqa: E402 +# Import cleanback package entrypoint +ROOT = Path(__file__).resolve().parents[2] # repo root +sys.path.insert(0, str(ROOT / "src")) +from cleanback import __main__ as main # noqa: E402 # Keep tests snappy but reliable: # - "timeout" dirs sleep 0.3s in fake dirval