@@ -0,0 +1,333 @@ | |||||
# Created by https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||||
# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm+all,intellij+all,python,virtualenv | |||||
### Intellij+all ### | |||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||||
# User-specific stuff | |||||
.idea/**/workspace.xml | |||||
.idea/**/tasks.xml | |||||
.idea/**/usage.statistics.xml | |||||
.idea/**/dictionaries | |||||
.idea/**/shelf | |||||
# AWS User-specific | |||||
.idea/**/aws.xml | |||||
# Generated files | |||||
.idea/**/contentModel.xml | |||||
# Sensitive or high-churn files | |||||
.idea/**/dataSources/ | |||||
.idea/**/dataSources.ids | |||||
.idea/**/dataSources.local.xml | |||||
.idea/**/sqlDataSources.xml | |||||
.idea/**/dynamic.xml | |||||
.idea/**/uiDesigner.xml | |||||
.idea/**/dbnavigator.xml | |||||
# Gradle | |||||
.idea/**/gradle.xml | |||||
.idea/**/libraries | |||||
# Gradle and Maven with auto-import | |||||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||||
# since they will be recreated, and may cause churn. Uncomment if using | |||||
# auto-import. | |||||
# .idea/artifacts | |||||
# .idea/compiler.xml | |||||
# .idea/jarRepositories.xml | |||||
# .idea/modules.xml | |||||
# .idea/*.iml | |||||
# .idea/modules | |||||
# *.iml | |||||
# *.ipr | |||||
# CMake | |||||
cmake-build-*/ | |||||
# Mongo Explorer plugin | |||||
.idea/**/mongoSettings.xml | |||||
# File-based project format | |||||
*.iws | |||||
# IntelliJ | |||||
out/ | |||||
# mpeltonen/sbt-idea plugin | |||||
.idea_modules/ | |||||
# JIRA plugin | |||||
atlassian-ide-plugin.xml | |||||
# Cursive Clojure plugin | |||||
.idea/replstate.xml | |||||
# SonarLint plugin | |||||
.idea/sonarlint/ | |||||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||||
com_crashlytics_export_strings.xml | |||||
crashlytics.properties | |||||
crashlytics-build.properties | |||||
fabric.properties | |||||
# Editor-based Rest Client | |||||
.idea/httpRequests | |||||
# Android studio 3.1+ serialized cache file | |||||
.idea/caches/build_file_checksums.ser | |||||
### Intellij+all Patch ### | |||||
# Ignore everything but code style settings and run configurations | |||||
# that are supposed to be shared within teams. | |||||
.idea/* | |||||
!.idea/codeStyles | |||||
!.idea/runConfigurations | |||||
### PyCharm+all ### | |||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||||
# User-specific stuff | |||||
# AWS User-specific | |||||
# Generated files | |||||
# Sensitive or high-churn files | |||||
# Gradle | |||||
# Gradle and Maven with auto-import | |||||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||||
# since they will be recreated, and may cause churn. Uncomment if using | |||||
# auto-import. | |||||
# .idea/artifacts | |||||
# .idea/compiler.xml | |||||
# .idea/jarRepositories.xml | |||||
# .idea/modules.xml | |||||
# .idea/*.iml | |||||
# .idea/modules | |||||
# *.iml | |||||
# *.ipr | |||||
# CMake | |||||
# Mongo Explorer plugin | |||||
# File-based project format | |||||
# IntelliJ | |||||
# mpeltonen/sbt-idea plugin | |||||
# JIRA plugin | |||||
# Cursive Clojure plugin | |||||
# SonarLint plugin | |||||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||||
# Editor-based Rest Client | |||||
# Android studio 3.1+ serialized cache file | |||||
### PyCharm+all Patch ### | |||||
# Ignore everything but code style settings and run configurations | |||||
# that are supposed to be shared within teams. | |||||
### Python ### | |||||
# Byte-compiled / optimized / DLL files | |||||
__pycache__/ | |||||
*.py[cod] | |||||
*$py.class | |||||
# C extensions | |||||
*.so | |||||
# Distribution / packaging | |||||
.Python | |||||
build/ | |||||
develop-eggs/ | |||||
dist/ | |||||
downloads/ | |||||
eggs/ | |||||
.eggs/ | |||||
lib/ | |||||
lib64/ | |||||
parts/ | |||||
sdist/ | |||||
var/ | |||||
wheels/ | |||||
share/python-wheels/ | |||||
*.egg-info/ | |||||
.installed.cfg | |||||
*.egg | |||||
MANIFEST | |||||
# PyInstaller | |||||
# Usually these files are written by a python script from a template | |||||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | |||||
*.manifest | |||||
*.spec | |||||
# Installer logs | |||||
pip-log.txt | |||||
pip-delete-this-directory.txt | |||||
# Unit test / coverage reports | |||||
htmlcov/ | |||||
.tox/ | |||||
.nox/ | |||||
.coverage | |||||
.coverage.* | |||||
.cache | |||||
nosetests.xml | |||||
coverage.xml | |||||
*.cover | |||||
*.py,cover | |||||
.hypothesis/ | |||||
.pytest_cache/ | |||||
cover/ | |||||
# Translations | |||||
*.mo | |||||
*.pot | |||||
# Django stuff: | |||||
*.log | |||||
local_settings.py | |||||
db.sqlite3 | |||||
db.sqlite3-journal | |||||
# Flask stuff: | |||||
instance/ | |||||
.webassets-cache | |||||
# Scrapy stuff: | |||||
.scrapy | |||||
# Sphinx documentation | |||||
docs/_build/ | |||||
# PyBuilder | |||||
.pybuilder/ | |||||
target/ | |||||
# Jupyter Notebook | |||||
.ipynb_checkpoints | |||||
# IPython | |||||
profile_default/ | |||||
ipython_config.py | |||||
# pyenv | |||||
# For a library or package, you might want to ignore these files since the code is | |||||
# intended to run in multiple environments; otherwise, check them in: | |||||
# .python-version | |||||
# pipenv | |||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. | |||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies | |||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not | |||||
# install all needed dependencies. | |||||
#Pipfile.lock | |||||
# poetry | |||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. | |||||
# This is especially recommended for binary packages to ensure reproducibility, and is more | |||||
# commonly ignored for libraries. | |||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control | |||||
#poetry.lock | |||||
# pdm | |||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. | |||||
#pdm.lock | |||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it | |||||
# in version control. | |||||
# https://pdm.fming.dev/#use-with-ide | |||||
.pdm.toml | |||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm | |||||
__pypackages__/ | |||||
# Celery stuff | |||||
celerybeat-schedule | |||||
celerybeat.pid | |||||
# SageMath parsed files | |||||
*.sage.py | |||||
# Environments | |||||
.env | |||||
.venv | |||||
env/ | |||||
venv/ | |||||
ENV/ | |||||
env.bak/ | |||||
venv.bak/ | |||||
# Spyder project settings | |||||
.spyderproject | |||||
.spyproject | |||||
# Rope project settings | |||||
.ropeproject | |||||
# mkdocs documentation | |||||
/site | |||||
# mypy | |||||
.mypy_cache/ | |||||
.dmypy.json | |||||
dmypy.json | |||||
# Pyre type checker | |||||
.pyre/ | |||||
# pytype static type analyzer | |||||
.pytype/ | |||||
# Cython debug symbols | |||||
cython_debug/ | |||||
# PyCharm | |||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can | |||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore | |||||
# and can be added to the global gitignore or merged into this file. For a more nuclear | |||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder. | |||||
#.idea/ | |||||
### Python Patch ### | |||||
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration | |||||
poetry.toml | |||||
# ruff | |||||
.ruff_cache/ | |||||
# LSP config files | |||||
pyrightconfig.json | |||||
### VirtualEnv ### | |||||
# Virtualenv | |||||
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ | |||||
[Bb]in | |||||
[Ii]nclude | |||||
[Ll]ib | |||||
[Ll]ib64 | |||||
[Ll]ocal | |||||
[Ss]cripts | |||||
pyvenv.cfg | |||||
pip-selfcheck.json | |||||
# End of https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||||
/venv | |||||
/config.py |
@@ -0,0 +1,18 @@ | |||||
--- | |||||
kind: pipeline | |||||
name: default | |||||
steps: | |||||
- name: docker | |||||
image: plugins/docker | |||||
settings: | |||||
registry: atdr-writer.meo.ws | |||||
username: | |||||
from_secret: atdr_user | |||||
password: | |||||
from_secret: atdr_pass | |||||
repo: atdr-writer.meo.ws/archiveteam/offload-dispatcher | |||||
dockerfile: Dockerfile | |||||
purge: true | |||||
auto_tag: false | |||||
tags: | |||||
- latest |
@@ -0,0 +1,333 @@ | |||||
# Created by https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||||
# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm+all,intellij+all,python,virtualenv | |||||
### Intellij+all ### | |||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||||
# User-specific stuff | |||||
.idea/**/workspace.xml | |||||
.idea/**/tasks.xml | |||||
.idea/**/usage.statistics.xml | |||||
.idea/**/dictionaries | |||||
.idea/**/shelf | |||||
# AWS User-specific | |||||
.idea/**/aws.xml | |||||
# Generated files | |||||
.idea/**/contentModel.xml | |||||
# Sensitive or high-churn files | |||||
.idea/**/dataSources/ | |||||
.idea/**/dataSources.ids | |||||
.idea/**/dataSources.local.xml | |||||
.idea/**/sqlDataSources.xml | |||||
.idea/**/dynamic.xml | |||||
.idea/**/uiDesigner.xml | |||||
.idea/**/dbnavigator.xml | |||||
# Gradle | |||||
.idea/**/gradle.xml | |||||
.idea/**/libraries | |||||
# Gradle and Maven with auto-import | |||||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||||
# since they will be recreated, and may cause churn. Uncomment if using | |||||
# auto-import. | |||||
# .idea/artifacts | |||||
# .idea/compiler.xml | |||||
# .idea/jarRepositories.xml | |||||
# .idea/modules.xml | |||||
# .idea/*.iml | |||||
# .idea/modules | |||||
# *.iml | |||||
# *.ipr | |||||
# CMake | |||||
cmake-build-*/ | |||||
# Mongo Explorer plugin | |||||
.idea/**/mongoSettings.xml | |||||
# File-based project format | |||||
*.iws | |||||
# IntelliJ | |||||
out/ | |||||
# mpeltonen/sbt-idea plugin | |||||
.idea_modules/ | |||||
# JIRA plugin | |||||
atlassian-ide-plugin.xml | |||||
# Cursive Clojure plugin | |||||
.idea/replstate.xml | |||||
# SonarLint plugin | |||||
.idea/sonarlint/ | |||||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||||
com_crashlytics_export_strings.xml | |||||
crashlytics.properties | |||||
crashlytics-build.properties | |||||
fabric.properties | |||||
# Editor-based Rest Client | |||||
.idea/httpRequests | |||||
# Android studio 3.1+ serialized cache file | |||||
.idea/caches/build_file_checksums.ser | |||||
### Intellij+all Patch ### | |||||
# Ignore everything but code style settings and run configurations | |||||
# that are supposed to be shared within teams. | |||||
.idea/* | |||||
!.idea/codeStyles | |||||
!.idea/runConfigurations | |||||
### PyCharm+all ### | |||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||||
# User-specific stuff | |||||
# AWS User-specific | |||||
# Generated files | |||||
# Sensitive or high-churn files | |||||
# Gradle | |||||
# Gradle and Maven with auto-import | |||||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||||
# since they will be recreated, and may cause churn. Uncomment if using | |||||
# auto-import. | |||||
# .idea/artifacts | |||||
# .idea/compiler.xml | |||||
# .idea/jarRepositories.xml | |||||
# .idea/modules.xml | |||||
# .idea/*.iml | |||||
# .idea/modules | |||||
# *.iml | |||||
# *.ipr | |||||
# CMake | |||||
# Mongo Explorer plugin | |||||
# File-based project format | |||||
# IntelliJ | |||||
# mpeltonen/sbt-idea plugin | |||||
# JIRA plugin | |||||
# Cursive Clojure plugin | |||||
# SonarLint plugin | |||||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||||
# Editor-based Rest Client | |||||
# Android studio 3.1+ serialized cache file | |||||
### PyCharm+all Patch ### | |||||
# Ignore everything but code style settings and run configurations | |||||
# that are supposed to be shared within teams. | |||||
### Python ### | |||||
# Byte-compiled / optimized / DLL files | |||||
__pycache__/ | |||||
*.py[cod] | |||||
*$py.class | |||||
# C extensions | |||||
*.so | |||||
# Distribution / packaging | |||||
.Python | |||||
build/ | |||||
develop-eggs/ | |||||
dist/ | |||||
downloads/ | |||||
eggs/ | |||||
.eggs/ | |||||
lib/ | |||||
lib64/ | |||||
parts/ | |||||
sdist/ | |||||
var/ | |||||
wheels/ | |||||
share/python-wheels/ | |||||
*.egg-info/ | |||||
.installed.cfg | |||||
*.egg | |||||
MANIFEST | |||||
# PyInstaller | |||||
# Usually these files are written by a python script from a template | |||||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | |||||
*.manifest | |||||
*.spec | |||||
# Installer logs | |||||
pip-log.txt | |||||
pip-delete-this-directory.txt | |||||
# Unit test / coverage reports | |||||
htmlcov/ | |||||
.tox/ | |||||
.nox/ | |||||
.coverage | |||||
.coverage.* | |||||
.cache | |||||
nosetests.xml | |||||
coverage.xml | |||||
*.cover | |||||
*.py,cover | |||||
.hypothesis/ | |||||
.pytest_cache/ | |||||
cover/ | |||||
# Translations | |||||
*.mo | |||||
*.pot | |||||
# Django stuff: | |||||
*.log | |||||
local_settings.py | |||||
db.sqlite3 | |||||
db.sqlite3-journal | |||||
# Flask stuff: | |||||
instance/ | |||||
.webassets-cache | |||||
# Scrapy stuff: | |||||
.scrapy | |||||
# Sphinx documentation | |||||
docs/_build/ | |||||
# PyBuilder | |||||
.pybuilder/ | |||||
target/ | |||||
# Jupyter Notebook | |||||
.ipynb_checkpoints | |||||
# IPython | |||||
profile_default/ | |||||
ipython_config.py | |||||
# pyenv | |||||
# For a library or package, you might want to ignore these files since the code is | |||||
# intended to run in multiple environments; otherwise, check them in: | |||||
# .python-version | |||||
# pipenv | |||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. | |||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies | |||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not | |||||
# install all needed dependencies. | |||||
#Pipfile.lock | |||||
# poetry | |||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. | |||||
# This is especially recommended for binary packages to ensure reproducibility, and is more | |||||
# commonly ignored for libraries. | |||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control | |||||
#poetry.lock | |||||
# pdm | |||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. | |||||
#pdm.lock | |||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it | |||||
# in version control. | |||||
# https://pdm.fming.dev/#use-with-ide | |||||
.pdm.toml | |||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm | |||||
__pypackages__/ | |||||
# Celery stuff | |||||
celerybeat-schedule | |||||
celerybeat.pid | |||||
# SageMath parsed files | |||||
*.sage.py | |||||
# Environments | |||||
.env | |||||
.venv | |||||
env/ | |||||
venv/ | |||||
ENV/ | |||||
env.bak/ | |||||
venv.bak/ | |||||
# Spyder project settings | |||||
.spyderproject | |||||
.spyproject | |||||
# Rope project settings | |||||
.ropeproject | |||||
# mkdocs documentation | |||||
/site | |||||
# mypy | |||||
.mypy_cache/ | |||||
.dmypy.json | |||||
dmypy.json | |||||
# Pyre type checker | |||||
.pyre/ | |||||
# pytype static type analyzer | |||||
.pytype/ | |||||
# Cython debug symbols | |||||
cython_debug/ | |||||
# PyCharm | |||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can | |||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore | |||||
# and can be added to the global gitignore or merged into this file. For a more nuclear | |||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder. | |||||
#.idea/ | |||||
### Python Patch ### | |||||
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration | |||||
poetry.toml | |||||
# ruff | |||||
.ruff_cache/ | |||||
# LSP config files | |||||
pyrightconfig.json | |||||
### VirtualEnv ### | |||||
# Virtualenv | |||||
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ | |||||
[Bb]in | |||||
[Ii]nclude | |||||
[Ll]ib | |||||
[Ll]ib64 | |||||
[Ll]ocal | |||||
[Ss]cripts | |||||
pyvenv.cfg | |||||
pip-selfcheck.json | |||||
# End of https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||||
/venv | |||||
/config.py |
@@ -0,0 +1,14 @@ | |||||
FROM python:3.11-bookworm | |||||
RUN apt update && apt install -y tini ca-certificates && \ | |||||
apt-get clean && \ | |||||
rm -rf /var/lib/apt/lists/* | |||||
WORKDIR /dispatcher | |||||
COPY requirements.txt . | |||||
RUN pip install -r requirements.txt | |||||
COPY main.py . | |||||
ENV DISPATCHER_SETTINGS=/dispatcher/config.py | |||||
ENTRYPOINT [ "/usr/bin/tini-static", "--", "gunicorn", "-w", "4", "--access-logfile=-", "main:app" ] |
@@ -0,0 +1,12 @@ | |||||
version: "3.7" | |||||
services: | |||||
dispatcher: | |||||
build: | |||||
context: . | |||||
volumes: | |||||
- './config.py:/dispatcher/config.py:ro' | |||||
ports: | |||||
- '127.0.0.1:8000:8000' | |||||
command: | |||||
- '-b' | |||||
- '0.0.0.0' |
@@ -0,0 +1,22 @@ | |||||
PROMETHEUS = { | |||||
"url": "http://127.0.0.1:9090", | |||||
"disable_ssl": True, | |||||
} | |||||
# URL FORMATS: | |||||
# | |||||
# Backend # TLS # Bucket/item # URL Format | |||||
########################################### | |||||
# Minio S3 # N # Y # minio+http://user:password@host[:port] | |||||
# Minio S3 # Y # Y # minio+https://user:password@host[:port] | |||||
TARGETS = [ | |||||
{ | |||||
"url": "minio+https://user:password@minio", | |||||
"free_space": { | |||||
"query": "minio_cluster_capacity_usable_free_bytes{job=\"somejob\"}", | |||||
"minimum": (1024 * 1024 * 1024 * 500), # 500 gigabytes | |||||
} | |||||
}, | |||||
] |
@@ -0,0 +1,38 @@ | |||||
import copy | |||||
from flask import Flask, request | |||||
from prometheus_api_client import PrometheusConnect | |||||
import random | |||||
app = Flask(__name__) | |||||
app.config.from_envvar('DISPATCHER_SETTINGS') | |||||
def get_prom_client(): | |||||
return PrometheusConnect(**app.config["PROMETHEUS"]) | |||||
@app.route("/") | |||||
def hello_world(): | |||||
return "Target offload dispatcher" | |||||
@app.route("/offload_target") | |||||
def offload_target(): | |||||
expected_size = int(request.args.get("SIZE_HINT", 20*1024*1024*1024)) | |||||
targets = copy.deepcopy(app.config["TARGETS"]) | |||||
random.shuffle(targets) | |||||
prom = get_prom_client() | |||||
for target in targets: | |||||
app.logger.info(f"Considering target {target['url']}...") | |||||
result = prom.custom_query(query=target["free_space"]["query"]) | |||||
if len(result) != 1: | |||||
continue | |||||
result = result[0] | |||||
target_free_space = int(result['value'][1]) | |||||
expected_free_space = target_free_space - expected_size | |||||
app.logger.info(f"Available space: {target_free_space} Expected size: {expected_size}") | |||||
if expected_free_space < target["free_space"].get("minimum", 1024 * 1024 * 1024 * 500): | |||||
continue | |||||
return {"url": target["url"]} | |||||
return "Unable to allocate target", 500 |
@@ -0,0 +1,3 @@ | |||||
flask~=2.3.3 | |||||
gunicorn | |||||
prometheus-api-client |