diff --git a/.tmp.qEgXmoMtcA/archiveteam/repositories.json b/.tmp.qEgXmoMtcA/archiveteam/repositories.json new file mode 100644 index 0000000000..e8d25b97b2 --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/repositories.json @@ -0,0 +1,20 @@ +[ + { + "user": "archiveteam", + "name": "warrior-dockerfile", + "namespace": "archiveteam", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": true, + "can_edit": false, + "star_count": 16, + "pull_count": 10898411, + "last_updated": "2018-12-17T00:42:59.374566Z", + "is_migrated": true, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "archiveteam" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/Dockerfile b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/Dockerfile new file mode 100644 index 0000000000..f81f4c30cd --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/Dockerfile @@ -0,0 +1,80 @@ +# Use phusion/baseimage as base image. +FROM phusion/baseimage:0.11 + +# Use baseimage-docker's init system. +CMD ["/sbin/my_init"] + +ADD get-wget-lua.sh / + +# Install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + autoconf \ + flex \ + gcc \ + git \ + isc-dhcp-client \ + jq \ + libgnutls28-dev \ + liblua5.1-0 \ + liblua5.1-0-dev \ + make \ + net-tools \ + pciutils \ + python \ + python-pip \ + python-setuptools \ + python3 \ + python3-pip \ + python3-setuptools \ + rsync \ + software-properties-common \ + sudo \ + wget \ + && chmod +x /get-wget-lua.sh && sync && bash -c "/get-wget-lua.sh" \ + && apt-get remove -y \ + autoconf \ + flex \ + gcc \ + libgnutls28-dev \ + liblua5.1-0-dev \ + make \ + && apt-get clean && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Setup system for the warrior +RUN useradd warrior +RUN mkdir /home/warrior && chown warrior: /home/warrior + +# Clone warrior code +USER warrior +WORKDIR /home/warrior +RUN git clone -b docker https://github.com/ArchiveTeam/warrior-code2.git +USER root + +# Add the boot script (this will install the actual warrior on boot) +RUN mkdir -p /etc/my_init.d +ADD boot.sh /etc/my_init.d/warrior-boot.sh + +# running as root and/or requiring sudo is a bad practice in docker containers, +# sadly sudo is hard-coded all over the place in `warrior-code2` +RUN echo "warrior ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +# Expose web interface port +EXPOSE 8001 + +# Add the warrior service entry for runit +RUN mkdir /etc/service/warrior +ADD warrior.sh /etc/service/warrior/run + +# ENV to JSON +ADD env-to-json.sh /home/warrior + +ENV DOWNLOADER="" +ENV HTTP_PASSWORD="" +ENV HTTP_USERNAME="" +ENV SELECTED_PROJECT="" +ENV SHARED_RSYNC_THREADS="" +ENV WARRIOR_ID="" +ENV CONCURRENT_ITEMS="" + diff --git a/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/info.json b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/info.json new file mode 100644 index 0000000000..87d741fff9 --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/info.json @@ -0,0 +1,25 @@ +{ + "user": "archiveteam", + "name": "warrior-dockerfile", + "namespace": "archiveteam", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": true, + "can_edit": false, + "star_count": 16, + "pull_count": 10898412, + "last_updated": "2018-12-17T00:42:59.374566Z", + "is_migrated": true, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "archiveteam", + "has_starred": false, + "full_description": "## A Dockerfile for the [ArchiveTeam Warrior](https://www.archiveteam.org/index.php?title=ArchiveTeam_Warrior)\n\"Warrior\"Docker\n\nBuild, run, grab the container IP and access the web interface on port 8001.\n\nAvailable as a Trusted Build on the index as [`archiveteam/warrior-dockerfile`](https://index.docker.io/u/archiveteam/warrior-dockerfile/) so you can just\n\n```\ndocker pull archiveteam/warrior-dockerfile\n# run without -d to follow the warrior install process\n# you will need to detach or stop-and-start the container.\n# use -p to bind port 8001 on the docker container\n# (default ip 172.17.0.x) to port 8001 on localhost.\ndocker run [-d] [-p 127.0.0.1:8001:8001] archiveteam/warrior-dockerfile\n```\n\nIf you prefer to just run the process in the background, and automatically start it again after machine reboot, use this instead:\n\n``` shell-interaction\ndocker run --detach \\\n --publish 127.0.0.1:8001:8001 \\\n --restart always \\\n archiveteam/warrior-dockerfile\n```\n\n\n### Configuration\n\n\n#### Manual Using the Web Interface\nTo access the web interface get the container IP from `docker inspect` and point your browser to `http://IP:8001`. If you are running this container on a headless machine, be sure to bind the docker container's port to a port on that machine (e.g. `-p 8001:8001`) so that you can access the web interface on your LAN.\n\nYou can stop and resume the Warrior with `docker stop` and `docker start`\n\n\n#### Using Environment Variables\n\nIf you don't mount a `projects.json` configuration, you can provide seed settings using\nenvironment variables. Once a `projects.json` file exists, environment variables\nwill be ignored. Please note: This is currently not available in the Raspberry PI image.\n\n##### Example:\n\n```shell\n docker run \\\n --detach \\\n --env DOWNLOADER=\"your name\" -e \\\n --env SELECTED_PROJECT=\"auto\" \\\n --publish 8001:8001 \\\n --restart always \\\n archiveteam/warrior-dockerfile\n```\n\n##### Mapping\n\n| ENV | JSON key | Example | Default |\n|----------------------|----------------------|-------------------|---------|\n| DOWNLOADER | downloader | | |\n| HTTP_PASSWORD | http_password | | |\n| HTTP_USERNAME | http_username | | |\n| SELECTED_PROJECT | selected_project | `auto`, `tumblr` | |\n| SHARED_RSYNC_THREADS | shared:rsync_threads | | `20` |\n| WARRIOR_ID | warrior_id | | |\n| CONCURRENT_ITEMS | concurrent_items | | `3` |\n\n## Raspberry Pi\nYou can build the container with the following command:\n``` shell-interaction\ndocker build --rm -t warrior-arm32v5:latest -f Dockerfile.raspberry .\n```\n\nThe image needs a place to store the downloaded data as well as its\nconfiguration. Say you have a location suitable at /var/local/warrior\nuse the command below, otherwise update the data and config.json paths.\n\nFirst, create an empty config.json if it doesn't exist. Otherwise when you\nmount the path with docker it will create it as a directory.\n``` shell-interaction\ntouch /var/local/warrior/config.json\n```\n\nNow start the container.\n``` shell-interaction\ndocker run \\\n\t--volume /var/local/warrior/data:/data/data \\\n\t--volume /var/local/warrior/config.json:/home/warrior/projects/config.json \\\n\t--publish 8001:8001 \\\n\t--restart always \\\n\twarrior-arm32v5:latest\n```\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/sources.json b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/sources.json new file mode 100644 index 0000000000..75fc8f4ff5 --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/d7c66b31-70d8-4d86-b08b-256bc3f48c47/" + ], + "channel": "Stable", + "image": "archiveteam/warrior-dockerfile", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "warrior-dockerfile", + "resource_uri": "/api/build/v1/source/db2598ef-71cc-4786-a83b-ac055d36ba67/", + "state": "Empty", + "uuid": "db2598ef-71cc-4786-a83b-ac055d36ba67" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags.json b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags.json new file mode 100644 index 0000000000..edbf2809a5 --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6731, + "id": 7530, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:742864ef616f0a80bdd162538964c43df677f0912716ebda44605063258a6d18", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 119940149, + "status": "active", + "last_pulled": "2021-02-21T12:13:02.919107Z", + "last_pushed": null + } + ], + "last_updated": "2018-12-17T00:42:59.092382Z", + "last_updater": 6731, + "last_updater_username": "archiveteam", + "name": "latest", + "repository": 4436, + "full_size": 119940149, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T12:13:02.919107Z", + "tag_last_pushed": "2018-12-17T00:42:59.092382Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json new file mode 100644 index 0000000000..e3ec60f1dc --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json @@ -0,0 +1,170 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:742864ef616f0a80bdd162538964c43df677f0912716ebda44605063258a6d18", + "layers": [ + { + "digest": "sha256:c64513b741452f95d8a147b69c30f403f6289542dd7b2b51dd8ba0cb35d0e08b", + "size": 31658876, + "instruction": "ADD file:4bb62bb05874068552efeb626f8b31b4a29f26d6cc8c7d7fc7ab4c1fdece957a in / " + }, + { + "digest": "sha256:01b8b12bad90b51d9f15dd4b63103ea6221b339ac3b3e75807c963e678f28624", + "size": 847, + "instruction": "/bin/sh -c set -xe \t\t&& echo '#!/bin/sh' > /usr/sbin/policy-rc.d \t&& echo 'exit 101' >> /usr/sbin/policy-rc.d \t&& chmod +x /usr/sbin/policy-rc.d \t\t&& dpkg-divert --local --rename --add /sbin/initctl \t&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \t&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \t\t&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \t\t&& echo 'DPkg::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' > /etc/apt/apt.conf.d/docker-clean \t&& echo 'APT::Update::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' >> /etc/apt/apt.conf.d/docker-clean \t&& echo 'Dir::Cache::pkgcache \"\"; Dir::Cache::srcpkgcache \"\";' >> /etc/apt/apt.conf.d/docker-clean \t\t&& echo 'Acquire::Languages \"none\";' > /etc/apt/apt.conf.d/docker-no-languages \t\t&& echo 'Acquire::GzipIndexes \"true\"; Acquire::CompressionTypes::Order:: \"gz\";' > /etc/apt/apt.conf.d/docker-gzip-indexes \t\t&& echo 'Apt::AutoRemove::SuggestsImportant \"false\";' > /etc/apt/apt.conf.d/docker-autoremove-suggests" + }, + { + "digest": "sha256:c5d85cf7a05fec99bb829db84dc5a21cc0aca569253f45d1ea10ca9e8a03fa9a", + "size": 468, + "instruction": "/bin/sh -c rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b6b268720157210d21bbe49f6112f815774e6d2a6144b14911749fadfdb034f0", + "size": 849, + "instruction": "/bin/sh -c sed -i 's/^#\\s*\\(deb.*universe\\)$/\\1/g' /etc/apt/sources.list" + }, + { + "digest": "sha256:e12192999ff18f01315563c63333d7c1059cd8e64dffe75fffe504b95eeb093c", + "size": 163, + "instruction": "/bin/sh -c mkdir -p /run/systemd && echo 'docker' > /run/systemd/container" + }, + { + "size": 0, + "instruction": " CMD [\"/bin/bash\"]" + }, + { + "size": 0, + "instruction": " MAINTAINER Phusion " + }, + { + "digest": "sha256:d39ece66b66711cd10d00ecca889e6d9471a6c746abd4b1864d5159396013b07", + "size": 15045, + "instruction": "COPY dir:4156d6d2c4b57f6457db9076a456c428e0bc9deee73f34aaba10f3045eedeb35 in /bd_build " + }, + { + "digest": "sha256:65599be66378ba1b7638eb7f0ff71dc10f857d45ad289e61627e1256b3c52945", + "size": 35933387, + "instruction": "/bin/sh -c /bd_build/prepare.sh && \t/bd_build/system_services.sh && \t/bd_build/utilities.sh && \t/bd_build/cleanup.sh" + }, + { + "size": 0, + "instruction": " ENV DEBIAN_FRONTEND=teletype LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8" + }, + { + "size": 0, + "instruction": " CMD [\"/sbin/my_init\"]" + }, + { + "size": 0, + "instruction": " CMD [\"/sbin/my_init\"]" + }, + { + "digest": "sha256:5ce76b79e2fd9cff49fd987ba4b49641b5899ba24fa91893a5fdf7450c5d77ed", + "size": 673, + "instruction": "ADD file:5d94870fb53583fde1ef15d0c418754c94d42d0a82bc7b53e3dc93386a6cc064 in / " + }, + { + "digest": "sha256:9e96699e9e5ea169fd670896e04b4da6d193c143e030b5440f0e0c57442234f7", + "size": 52162287, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends autoconf flex gcc git isc-dhcp-client jq libgnutls28-dev liblua5.1-0 liblua5.1-0-dev make net-tools pciutils python python-pip python-setuptools python3 python3-pip python3-setuptools rsync software-properties-common sudo wget && chmod +x /get-wget-lua.sh && sync && bash -c \"/get-wget-lua.sh\" && apt-get remove -y autoconf flex gcc libgnutls28-dev liblua5.1-0-dev make && apt-get clean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*" + }, + { + "digest": "sha256:29870fd4e7bd112857984e8febf2d4c159b2088dcdac2df482073f76da9c3663", + "size": 2017, + "instruction": "/bin/sh -c useradd warrior" + }, + { + "digest": "sha256:29239dfdf917ce262ce161c99dfc4771868480db2fa98e64b61ad3513c898c78", + "size": 125, + "instruction": "/bin/sh -c mkdir /home/warrior && chown warrior: /home/warrior" + }, + { + "size": 0, + "instruction": " USER warrior" + }, + { + "size": 0, + "instruction": "WORKDIR /home/warrior" + }, + { + "digest": "sha256:99bc5d955aa52a839127a03297349b0205954c88b13cbc572c3d754f294e7b68", + "size": 163573, + "instruction": "/bin/sh -c git clone -b docker https://github.com/ArchiveTeam/warrior-code2.git" + }, + { + "size": 0, + "instruction": " USER root" + }, + { + "size": 0, + "instruction": "/bin/sh -c mkdir -p /etc/my_init.d" + }, + { + "digest": "sha256:b15ca0c6a79bf7c837e68bdb569e60f1d9c50372dd0c79a4bae6986b857cdd66", + "size": 259, + "instruction": "ADD file:47d258ef4e4ea1a817763d1496002fc6f8ddf531a760e99b078816f3196fccd6 in /etc/my_init.d/warrior-boot.sh " + }, + { + "digest": "sha256:6b59ed47645d2520c1677fdfe1ad21b75bc782518bd09f2125a3fa784a1bc353", + "size": 539, + "instruction": "/bin/sh -c echo \"warrior ALL=(ALL) NOPASSWD: ALL\" >> /etc/sudoers" + }, + { + "size": 0, + "instruction": " EXPOSE 8001" + }, + { + "digest": "sha256:a0be2c0f04b4ee626df02c7dd6d4379a10e6504a1f99f1737e9a9a96ef26f991", + "size": 184, + "instruction": "/bin/sh -c mkdir /etc/service/warrior" + }, + { + "digest": "sha256:c93d3f659e1baeb1bce005a4e692b95db8db3574f415a1c01133d1779136650e", + "size": 349, + "instruction": "ADD file:6a76fa03c473181539860b5a1806568aa61928c6fa5f0950e88306616d576d3c in /etc/service/warrior/run " + }, + { + "digest": "sha256:d80eda2ef6b14c973c0372a8f706de70324fb7a531196e7a3bd2e0c978b64f4e", + "size": 508, + "instruction": "ADD file:48a7d65247d6ff3054a26127cbe7816687abbc79b682b0442b660e7480187a6e in /home/warrior " + }, + { + "size": 0, + "instruction": " ENV DOWNLOADER=" + }, + { + "size": 0, + "instruction": " ENV HTTP_PASSWORD=" + }, + { + "size": 0, + "instruction": " ENV HTTP_USERNAME=" + }, + { + "size": 0, + "instruction": " ENV SELECTED_PROJECT=" + }, + { + "size": 0, + "instruction": " ENV SHARED_RSYNC_THREADS=" + }, + { + "size": 0, + "instruction": " ENV WARRIOR_ID=" + }, + { + "size": 0, + "instruction": " ENV CONCURRENT_ITEMS=" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 119940149, + "status": "active", + "last_pulled": "2021-02-21T12:13:02.919107Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json new file mode 100644 index 0000000000..00cff41ef0 --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6731, + "id": 7530, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:742864ef616f0a80bdd162538964c43df677f0912716ebda44605063258a6d18", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 119940149, + "status": "active", + "last_pulled": "2021-02-21T12:13:02.919107Z", + "last_pushed": null + } + ], + "last_updated": "2018-12-17T00:42:59.092382Z", + "last_updater": 6731, + "last_updater_username": "archiveteam", + "name": "latest", + "repository": 4436, + "full_size": 119940149, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T12:13:02.919107Z", + "tag_last_pushed": "2018-12-17T00:42:59.092382Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/archiveteam/user.json b/.tmp.qEgXmoMtcA/archiveteam/user.json new file mode 100644 index 0000000000..5a05b70db9 --- /dev/null +++ b/.tmp.qEgXmoMtcA/archiveteam/user.json @@ -0,0 +1,11 @@ +{ + "id": "2d4b1e3074c011e4bea40242ac11001b", + "username": "archiveteam", + "full_name": "", + "location": "", + "company": "", + "profile_url": "", + "date_joined": "2013-11-29T04:49:43Z", + "gravatar_url": "https://secure.gravatar.com/avatar/bd6261f1b7318c3d63261e32b218dc5a.jpg?s=80&r=g&d=mm", + "type": "User" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/dockerhub-metadata.retrieve.log b/.tmp.qEgXmoMtcA/dockerhub-metadata.retrieve.log new file mode 100644 index 0000000000..76e63076cd --- /dev/null +++ b/.tmp.qEgXmoMtcA/dockerhub-metadata.retrieve.log @@ -0,0 +1,135 @@ +2021-02-21T13:00:03.079240Z Retrieving Docker Hub user archiveteam +2021-02-21T13:00:03.083607Z Fetching https://hub.docker.com/v2/users/archiveteam/ +2021-02-21T13:00:04.115094Z Fetching https://hub.docker.com/v2/repositories/archiveteam/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:05.024796Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/ +2021-02-21T13:00:05.946967Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/dockerfile/ +2021-02-21T13:00:06.543354Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:07.522440Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:08.115911Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/ +2021-02-21T13:00:08.699607Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/images +2021-02-21T13:00:08.734837Z Done with archiveteam +2021-02-21T13:00:08.735754Z Retrieving Docker Hub user warcforceone +2021-02-21T13:00:08.735910Z Fetching https://hub.docker.com/v2/users/warcforceone/ +2021-02-21T13:00:09.542362Z Fetching https://hub.docker.com/v2/repositories/warcforceone/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:10.394904Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/ +2021-02-21T13:00:11.055060Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/dockerfile/ +2021-02-21T13:00:11.634541Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:12.565512Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:13.164666Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/ +2021-02-21T13:00:13.739402Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/images +2021-02-21T13:00:14.318771Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/ +2021-02-21T13:00:14.933416Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/dockerfile/ +2021-02-21T13:00:15.533648Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:16.480206Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:17.047920Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/ +2021-02-21T13:00:17.620711Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/images +2021-02-21T13:00:18.191312Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/ +2021-02-21T13:00:18.798070Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/dockerfile/ +2021-02-21T13:00:19.402354Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:20.329141Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:20.916344Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/ +2021-02-21T13:00:21.509165Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/images +2021-02-21T13:00:22.084129Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/ +2021-02-21T13:00:22.713104Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/dockerfile/ +2021-02-21T13:00:23.305233Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:24.229953Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:24.796820Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/ +2021-02-21T13:00:25.360603Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/images +2021-02-21T13:00:25.932798Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/ +2021-02-21T13:00:26.573506Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/dockerfile/ +2021-02-21T13:00:27.158402Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:28.099552Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:28.697193Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/ +2021-02-21T13:00:29.278380Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/images +2021-02-21T13:00:29.846497Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/ +2021-02-21T13:00:30.451672Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/dockerfile/ +2021-02-21T13:00:31.039122Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:31.960825Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:32.565245Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/ +2021-02-21T13:00:33.140830Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/images +2021-02-21T13:00:33.727926Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/ +2021-02-21T13:00:34.341345Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/dockerfile/ +2021-02-21T13:00:34.939264Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:35.989032Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:36.582377Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/ +2021-02-21T13:00:38.289608Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/images +2021-02-21T13:00:38.897133Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/ +2021-02-21T13:00:39.546091Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/dockerfile/ +2021-02-21T13:00:40.148483Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:41.127927Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:41.713867Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/ +2021-02-21T13:00:42.287291Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/images +2021-02-21T13:00:42.860535Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/ +2021-02-21T13:00:43.487069Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/dockerfile/ +2021-02-21T13:00:44.087166Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:44.980044Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:45.551923Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/ +2021-02-21T13:00:46.174822Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/images +2021-02-21T13:00:46.748645Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/ +2021-02-21T13:00:47.372970Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/dockerfile/ +2021-02-21T13:00:47.963279Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:48.880807Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:49.466401Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/ +2021-02-21T13:00:50.037459Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/images +2021-02-21T13:00:50.610600Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/ +2021-02-21T13:00:51.218546Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/dockerfile/ +2021-02-21T13:00:51.800509Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:52.849648Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:53.426923Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/ +2021-02-21T13:00:54.011048Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/images +2021-02-21T13:00:54.576517Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/ +2021-02-21T13:00:55.207153Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/dockerfile/ +2021-02-21T13:00:55.791025Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:00:56.718978Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:00:57.284751Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/ +2021-02-21T13:00:57.874617Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/images +2021-02-21T13:00:58.447921Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/ +2021-02-21T13:00:59.060076Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/dockerfile/ +2021-02-21T13:00:59.650835Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:01:00.630975Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:01:01.208830Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/ +2021-02-21T13:01:01.782573Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/images +2021-02-21T13:01:02.362415Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/ +2021-02-21T13:01:03.024074Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/dockerfile/ +2021-02-21T13:01:03.664821Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:01:04.561705Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:01:05.135281Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/ +2021-02-21T13:01:05.729177Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/images +2021-02-21T13:01:06.311657Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/ +2021-02-21T13:01:06.919059Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/dockerfile/ +2021-02-21T13:01:07.509035Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:01:09.721005Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:01:10.299070Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/ +2021-02-21T13:01:10.890349Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/images +2021-02-21T13:01:11.453768Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/ +2021-02-21T13:01:12.083085Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/dockerfile/ +2021-02-21T13:01:12.672672Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:01:13.401850Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:01:13.971508Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/ +2021-02-21T13:01:14.546394Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/images +2021-02-21T13:01:15.132959Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/ +2021-02-21T13:01:15.738211Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/dockerfile/ +2021-02-21T13:01:16.328701Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:01:17.291509Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:01:17.877604Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/ +2021-02-21T13:01:18.457147Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/images +2021-02-21T13:01:19.043721Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/ +2021-02-21T13:01:19.642525Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/dockerfile/ +2021-02-21T13:01:20.220940Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:01:20.891623Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:01:21.468437Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/ +2021-02-21T13:01:22.044626Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/images +2021-02-21T13:01:22.627935Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/ +2021-02-21T13:01:23.223579Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/dockerfile/ +2021-02-21T13:01:23.805309Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T13:01:24.736463Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T13:01:25.311463Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/ +2021-02-21T13:01:25.885692Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/images +Traceback (most recent call last): + File "/home/atdockerhub/dockerhub-metadata-master/retrieve", line 151, in + main() + File "/home/atdockerhub/dockerhub-metadata-master/retrieve", line 144, in main + retrieve(profile) + File "/home/atdockerhub/dockerhub-metadata-master/retrieve", line 134, in retrieve + json.dump(r.json(), fp, indent = '\t') +OSError: [Errno 28] No space left on device diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories.json b/.tmp.qEgXmoMtcA/warcforceone/repositories.json new file mode 100644 index 0000000000..7ad08add03 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories.json @@ -0,0 +1,668 @@ +[ + { + "user": "warcforceone", + "name": "dnsmasq", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 21054150, + "last_updated": "2021-02-20T00:18:15.003220Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "angelfire-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 412, + "last_updated": "2021-02-13T11:34:56.603218Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "grab-base", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 50530, + "last_updated": "2021-02-13T11:25:52.346081Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "google-sites-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7181220, + "last_updated": "2021-02-03T14:12:19.864923Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "github-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 13726771, + "last_updated": "2021-02-03T13:48:21.932838Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "pastebin-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 386443, + "last_updated": "2021-02-03T13:42:13.640186Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "reddit-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 5792737, + "last_updated": "2021-01-14T00:56:43.223341Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "domains-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 151168, + "last_updated": "2021-01-09T14:53:49.620078Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "megawarc-factory", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 80, + "last_updated": "2020-12-29T13:44:29.663201Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "fotoalbum-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 835123, + "last_updated": "2020-10-31T11:11:27.569458Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "youtube-playlistnotes-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11642977, + "last_updated": "2020-10-31T11:04:04.603216Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "mercurial-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 1139185, + "last_updated": "2020-10-31T10:57:10.413100Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "tencent-weibo-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 94271733, + "last_updated": "2020-10-31T00:33:22.019703Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "theartistunion-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 139872, + "last_updated": "2020-10-15T14:40:26.351961Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "bitbucket-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 976507, + "last_updated": "2020-10-15T14:32:38.301135Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "ateam-airsync", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 149, + "last_updated": "2020-09-29T20:52:21.216962Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "naver-matome-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 8478031, + "last_updated": "2020-09-22T15:42:01.244574Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "ateam-airsync-meta", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 197626, + "last_updated": "2020-09-04T01:36:11.426772Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "clutch-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11078089, + "last_updated": "2020-08-20T14:53:12.542994Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "clutchvideos-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 6455998, + "last_updated": "2020-08-15T01:12:03.411902Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "microsoft-download-center-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7473694, + "last_updated": "2020-08-03T20:33:39.400223Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "mixer-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 1299654, + "last_updated": "2020-07-23T17:26:35.210553Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "soup-io-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 935469, + "last_updated": "2020-07-21T16:00:50.757421Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "snscrape-upstream", + "namespace": "warcforceone", + "repository_type": null, + "status": 0, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 0, + "last_updated": null, + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "snscrape", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 21, + "last_updated": "2020-06-03T01:31:31.271076Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "singstar-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 23644, + "last_updated": "2020-01-21T23:35:49.829575Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "yahoogroups-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 251333, + "last_updated": "2019-12-12T17:52:18.779336Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "playstv-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 54289, + "last_updated": "2019-12-12T17:09:45.546166Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "youtube-likedlists-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 1, + "pull_count": 1489988, + "last_updated": "2019-12-09T23:22:16.580382Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "gfycat-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 491053, + "last_updated": "2019-11-28T23:21:38.456350Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "drawr-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 18578, + "last_updated": "2019-11-20T08:53:55.615761Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "radio24syv-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 56669, + "last_updated": "2019-11-02T21:19:03.786842Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "yourshot-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 28284, + "last_updated": "2019-10-30T01:49:43.816568Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "sketch-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 407761, + "last_updated": "2019-09-25T15:13:33.634433Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "tinypic-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7330350, + "last_updated": "2019-09-09T18:11:17.375929Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "freeml-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 22, + "last_updated": "2019-09-02T21:47:22.366328Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "instaudio-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 101, + "last_updated": "2019-08-03T02:38:42.937172Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/Dockerfile new file mode 100644 index 0000000000..c9b2c8d6ef --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-lua diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/info.json new file mode 100644 index 0000000000..10e5c88716 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "angelfire-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 412, + "last_updated": "2021-02-13T11:34:56.603218Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "angelfire-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [angelfire](http://archiveteam.org/index.php?title=angelfire)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"angelfire\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n pip install --upgrade seesaw\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/angelfire-grab.git; cd angelfire-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/angelfire-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y install autoconf automake flex gnutls-devel lua-devel python-pip zlib-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, angelfire-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/angelfire-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/sources.json new file mode 100644 index 0000000000..760acdfe5a --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/5d60025b-29a9-461d-8285-0525769b886a/" + ], + "channel": "Stable", + "image": "warcforceone/angelfire-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "angelfire-grab", + "resource_uri": "/api/build/v1/source/0346b47f-beb6-4bb1-b437-dfac7e86521c/", + "state": "Success", + "uuid": "0346b47f-beb6-4bb1-b437-dfac7e86521c" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags.json new file mode 100644 index 0000000000..9ffb8b8071 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 63790251, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b512078401ce1016c06d80a6dcf38589d4b1d3f2faf42078bf0d39bbe8e0d32a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 86392705, + "status": "active", + "last_pulled": "2021-02-20T02:21:56.8533Z", + "last_pushed": "2021-02-13T11:34:56.278689Z" + } + ], + "last_updated": "2021-02-13T11:34:56.278689Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470931, + "full_size": 86392705, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:56.8533Z", + "tag_last_pushed": "2021-02-13T11:34:56.278689Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags/latest.images.json new file mode 100644 index 0000000000..a1b54f3a72 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags/latest.images.json @@ -0,0 +1,115 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:b512078401ce1016c06d80a6dcf38589d4b1d3f2faf42078bf0d39bbe8e0d32a", + "layers": [ + { + "digest": "sha256:45b42c59be334ecda0daaa139b2f7d310e45c564c5f12263b1b8e68ec9e810ed", + "size": 27095142, + "instruction": "ADD file:d5c41bfaf15180481d8606f50799297e3f49b8a258c7c2cd988ab2bf0013272d in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:8cd3485318dba8e325f6b665105e10c51f4011cf2e2b4378ae2b92e104ff3566", + "size": 2769103, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tnetbase \t\ttzdata \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:b430e1981978aa8083832db9dc3f927e4336fbccf20f27a0419a5d31f78205b7", + "size": 10888464, + "instruction": "/bin/sh -c set -ex \t\t&& savedAptMark=\"$(apt-mark showmanual)\" \t&& apt-get update && apt-get install -y --no-install-recommends \t\tdpkg-dev \t\tgcc \t\tlibbluetooth-dev \t\tlibbz2-dev \t\tlibc6-dev \t\tlibexpat1-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tliblzma-dev \t\tlibncursesw5-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tmake \t\ttk-dev \t\tuuid-dev \t\twget \t\txz-utils \t\tzlib1g-dev \t\t$(command -v gpg > /dev/null || echo 'gnupg dirmngr') \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tLDFLAGS=\"-Wl,--strip-all\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& apt-mark auto '.*' > /dev/null \t&& apt-mark manual $savedAptMark \t&& find /usr/local -type f -executable -not \\( -name '*tkinter*' \\) -exec ldd '{}' ';' \t\t| awk '/=>/ { print $(NF-1) }' \t\t| sort -u \t\t| xargs -r dpkg-query --search \t\t| cut -d: -f1 \t\t| sort -u \t\t| xargs -r apt-mark manual \t&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \t&& rm -rf /var/lib/apt/lists/* \t\t&& python3 --version" + }, + { + "digest": "sha256:37d877dc41c386154662483278f5b856d8c34e200d571eb41cee562506fa5bfb", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=21.0.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/4be3fe44ad9dedc028629ed1497052d65d281b8e/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=8006625804f55e1bd99ad4214fd07082fee27a1c35945648a58f9087a714e9d4" + }, + { + "digest": "sha256:0b70d4ba14ef92ccaa7839657e6fca3422f09041c50a44011d6a2382d0c23fe8", + "size": 2452248, + "instruction": "/bin/sh -c set -ex; \t\tsavedAptMark=\"$(apt-mark showmanual)\"; \tapt-get update; \tapt-get install -y --no-install-recommends wget; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tapt-mark auto '.*' > /dev/null; \t[ -z \"$savedAptMark\" ] || apt-mark manual $savedAptMark; \tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \trm -rf /var/lib/apt/lists/*; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "digest": "sha256:2a5987f3c885013cfbe6ee2d5fa8bf80cbc61323e0ac8aa63ab2720faa2b9531", + "size": 926625, + "instruction": "COPY file:c7a381cd8311209dda148a7d15ef252c496da848907afaa18f8371d09648021e in /usr/local/bin/wget-lua " + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:4c4e90548af1cdbeb6d8ce5cde1c907ac0a4e48aed698d9a80b2afd3d3128111", + "size": 41771879, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket libpsl5 git && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0c0bb03728b9b8735afce4e3653c7761db07f3245cfa66a44bda45f8022184c8", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:209ae7f910171062848d7b09e65cbfbae13be920960f918f7ffdc862ff777905", + "size": 488779, + "instruction": "COPY dir:4d7664ff6cefb66182fe887cf6427bbf680a6a2d23fa04b10c3acc01deba141c in /grab " + }, + { + "digest": "sha256:29ab74bab7311b2ab6d658aa350e38f2b151ff058f507f36c89668ee9f35ef80", + "size": 140, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-lua" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 86392705, + "status": "active", + "last_pulled": "2021-02-20T02:21:56.8533Z", + "last_pushed": "2021-02-13T11:34:56.278689Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags/latest.info.json new file mode 100644 index 0000000000..4b7321c402 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/angelfire-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 63790251, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b512078401ce1016c06d80a6dcf38589d4b1d3f2faf42078bf0d39bbe8e0d32a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 86392705, + "status": "active", + "last_pulled": "2021-02-20T02:21:56.8533Z", + "last_pushed": "2021-02-13T11:34:56.278689Z" + } + ], + "last_updated": "2021-02-13T11:34:56.278689Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470931, + "full_size": 86392705, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:56.8533Z", + "tag_last_pushed": "2021-02-13T11:34:56.278689Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/info.json new file mode 100644 index 0000000000..57f909b416 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "ateam-airsync-meta", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 197626, + "last_updated": "2020-09-04T01:36:11.426772Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/sources.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/sources.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags.json new file mode 100644 index 0000000000..f74304ca95 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 1202997, + "id": 115894726, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:47fd629d3787d565cd20db1a66d76c08667a41c2c03daddabbc260eb4840cb48", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 30305756, + "status": "active", + "last_pulled": "2021-01-24T11:47:34.569491Z", + "last_pushed": null + } + ], + "last_updated": "2020-09-04T01:36:11.0808Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 9759263, + "full_size": 30305756, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-01-24T11:47:34.569491Z", + "tag_last_pushed": "2020-09-04T01:36:11.0808Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags/latest.images.json new file mode 100644 index 0000000000..e011c3fe03 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags/latest.images.json @@ -0,0 +1,62 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:47fd629d3787d565cd20db1a66d76c08667a41c2c03daddabbc260eb4840cb48", + "layers": [ + { + "digest": "sha256:e7c96db7181be991f19a9fb6975cdbbd73c65f4a2681348e63a141a2192a5f10", + "size": 2757034, + "instruction": "ADD file:a86aea1f3a7d68f6ae03397b99ea77f2e9ee901c5c59e59f76f93adbb4035913 in / " + }, + { + "size": 0, + "instruction": " CMD [\"/bin/sh\"]" + }, + { + "size": 0, + "instruction": " ENV NODE_VERSION=12.2.0" + }, + { + "digest": "sha256:a9b145f64bbe7efacd482b8576a2098dd7054c7d612a9ab0ab06570b4ae2bb0d", + "size": 22472612, + "instruction": "/bin/sh -c addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node && apk add --no-cache libstdc++ && apk add --no-cache --virtual .build-deps binutils-gold curl g++ gcc gnupg libgcc linux-headers make python && for key in 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 FD3A5288F042B6850C66B31F09FE44734EB7990E 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 DD8F2338BAE7501E3DD5AC78C273792F7D83545D C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 B9AE9905FFD7803F25714661B63B535A4C206CA9 77984A986EBC2AA786BC0F66B01FBB92821C587A 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 4ED778F539E3634C779C87C6D7062848A1AB005C A48C2BEE680E841632CD4E44F07496B3EB3C1762 B9E2F5981AA6E0CD28160D9FF13993A75599653C ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz\" && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc\" && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && grep \" node-v$NODE_VERSION.tar.xz\\$\" SHASUMS256.txt | sha256sum -c - && tar -xf \"node-v$NODE_VERSION.tar.xz\" && cd \"node-v$NODE_VERSION\" && ./configure && make -j$(getconf _NPROCESSORS_ONLN) V= && make install && apk del .build-deps && cd .. && rm -Rf \"node-v$NODE_VERSION\" && rm \"node-v$NODE_VERSION.tar.xz\" SHASUMS256.txt.asc SHASUMS256.txt" + }, + { + "size": 0, + "instruction": " ENV YARN_VERSION=1.15.2" + }, + { + "digest": "sha256:3bcb5e14be535bd6a0d2366436d93d11b56bf9340365d25eb98f1397aec23470", + "size": 1333853, + "instruction": "/bin/sh -c apk add --no-cache --virtual .build-deps-yarn curl gnupg tar && for key in 6A010C5166006599AA17F08146C2130DFD2497F5 ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz\" && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc\" && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && mkdir -p /opt && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && apk del .build-deps-yarn" + }, + { + "size": 0, + "instruction": " CMD [\"node\"]" + }, + { + "digest": "sha256:de4ba1e42d5621c17db371c4c92fd649aaf691be6a5af0b327d617a01191f5af", + "size": 3741240, + "instruction": "/bin/sh -c apk upgrade --no-cache && apk add --no-cache bash coreutils" + }, + { + "digest": "sha256:f093645e5eb6f0884bb1f5d981582ea0ebd64322095e11c71950659d001000fb", + "size": 1017, + "instruction": "COPY file:e28d1f2938be3ca1f4b959a73d71e4d4b49fe3c5d7975bcd58a06511906ddbcf in / " + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"/index.js\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 30305756, + "status": "active", + "last_pulled": "2021-01-24T11:47:34.569491Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags/latest.info.json new file mode 100644 index 0000000000..9836b6e482 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync-meta/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 1202997, + "id": 115894726, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:47fd629d3787d565cd20db1a66d76c08667a41c2c03daddabbc260eb4840cb48", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 30305756, + "status": "active", + "last_pulled": "2021-01-24T11:47:34.569491Z", + "last_pushed": null + } + ], + "last_updated": "2020-09-04T01:36:11.0808Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 9759263, + "full_size": 30305756, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-01-24T11:47:34.569491Z", + "tag_last_pushed": "2020-09-04T01:36:11.0808Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/info.json new file mode 100644 index 0000000000..19e7bb048c --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "ateam-airsync", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 149, + "last_updated": "2020-09-29T20:52:21.216962Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/sources.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/sources.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags.json new file mode 100644 index 0000000000..0745f3549b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 1202997, + "id": 62496068, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:8b8a5cef7bc416c719b7276affbd7555ae22d5721cb905e8d191fe00b0b951e3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 21943646, + "status": "stale", + "last_pulled": "2020-11-11T18:43:58.252311Z", + "last_pushed": "2020-09-29T20:52:20.774509Z" + } + ], + "last_updated": "2020-09-29T20:52:20.774509Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 7415504, + "full_size": 21943646, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-11T18:43:58.252311Z", + "tag_last_pushed": "2020-09-29T20:52:20.774509Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags/latest.images.json new file mode 100644 index 0000000000..67bb172938 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags/latest.images.json @@ -0,0 +1,40 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:8b8a5cef7bc416c719b7276affbd7555ae22d5721cb905e8d191fe00b0b951e3", + "layers": [ + { + "digest": "sha256:5d2415897100578ccbd3a0f03c205c3554ebe0ce4068a6969f7017d7dc0b1417", + "size": 2797239, + "instruction": "ADD file:a4192f640e3909ff3d4886840e28e983cb331b6d23219e6fb5a90471b7def94f in / " + }, + { + "size": 0, + "instruction": " CMD [\"/bin/sh\"]" + }, + { + "digest": "sha256:ca76a6be0a7e5963ec329fec39fa977888c6ea5292a600ce91223fb15f9176ca", + "size": 19145172, + "instruction": "/bin/sh -c apk upgrade --no-cache && apk add --no-cache rsync supervisor bash coreutils findutils" + }, + { + "digest": "sha256:c6ef8d11a87ba8cc65f39a8a7b51b60d52fd1a1aed1f6a24abf7f2a09b6dd904", + "size": 1235, + "instruction": "COPY dir:51b32cc696da2426e94bc769e0b5a75362f54388bacc9ab6ee15e4245cec4d3e in / " + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"supervisord\" \"-nkc/etc/supervisord.conf\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 21943646, + "status": "stale", + "last_pulled": "2020-11-11T18:43:58.252311Z", + "last_pushed": "2020-09-29T20:52:20.774509Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags/latest.info.json new file mode 100644 index 0000000000..8baad2de0b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/ateam-airsync/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 1202997, + "id": 62496068, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:8b8a5cef7bc416c719b7276affbd7555ae22d5721cb905e8d191fe00b0b951e3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 21943646, + "status": "stale", + "last_pulled": "2020-11-11T18:43:58.252311Z", + "last_pushed": "2020-09-29T20:52:20.774509Z" + } + ], + "last_updated": "2020-09-29T20:52:20.774509Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 7415504, + "full_size": 21943646, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-11T18:43:58.252311Z", + "tag_last_pushed": "2020-09-29T20:52:20.774509Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/Dockerfile new file mode 100644 index 0000000000..ec760688d1 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM warcforceone/grab-base +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua \ + && chmod +x /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/info.json new file mode 100644 index 0000000000..e679a8cf48 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "bitbucket-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 976507, + "last_updated": "2020-10-15T14:32:38.301135Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "bitbucket-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Bitbucket](http://archiveteam.org/index.php?title=Bitbucket)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Bitbucket\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n pip install --upgrade seesaw zstandard\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/bitbucket-grab.git; cd bitbucket-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/bitbucket-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y install autoconf automake flex gnutls-devel lua-devel python-pip zlib-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, bitbucket-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/bitbucket-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/bitbucket-grab\n cd bitbucket-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #kickthebucket.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/sources.json new file mode 100644 index 0000000000..973b594101 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/8c7a76f2-4372-4093-a4c0-726c9ddce274/" + ], + "channel": "Stable", + "image": "warcforceone/bitbucket-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "bitbucket-grab", + "resource_uri": "/api/build/v1/source/4d235ec0-f970-4725-8029-205a4d1c3a37/", + "state": "Failed", + "uuid": "4d235ec0-f970-4725-8029-205a4d1c3a37" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags.json new file mode 100644 index 0000000000..62be68fed3 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 105714713, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:0205a9dfcd1de546e61e54216a0ea20b549831e605f2207d9fd25976dbe8914e", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368879851, + "status": "stale", + "last_pulled": "2021-01-12T17:24:17.88938Z", + "last_pushed": "2020-10-15T14:32:37.681063Z" + } + ], + "last_updated": "2020-10-15T14:32:37.681063Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9299518, + "full_size": 368879851, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-12T17:24:17.88938Z", + "tag_last_pushed": "2020-10-15T14:32:37.681063Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags/latest.images.json new file mode 100644 index 0000000000..529dd0a02f --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:0205a9dfcd1de546e61e54216a0ea20b549831e605f2207d9fd25976dbe8914e", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/fa7dc83944936bf09a0e4cb5d5ec852c0d256599/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6e0bb0a2c2533361d7f297ed547237caf1b7507f197835974c0dd7eba998c53c" + }, + { + "digest": "sha256:3eb2f327eb62905a17b993e9807eeb1c85f9c0172b0721b089586c9fdd4acaee", + "size": 2119480, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:0a47d90c3940a5dd80561f5659e27a5a7f8f7675d95420319ca74039ed94a671", + "size": 15164957, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b538559a01f82dea3b4dfad9ae58884da104ac09290b66f24874efe1a1dd458b", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:7a4090d986ab6ba276281b40ac7957d8206dcfc019a7df78a60086cca298403f", + "size": 14111449, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:c4a8fbe3409cd4f88f4e2671a6e590b3a6aeab77a9fb3c681d6ec758a9184450", + "size": 68666, + "instruction": "COPY dir:a1bb2ce0bcd0c1144c29cd12956892db1bfd40c9e0777f7515adbf0203e0ee88 in /grab " + }, + { + "digest": "sha256:ae5d978a3c8c3753c4c475547a48cbbcb53a3924133cdaa1e2e6f5d2b6fcfb24", + "size": 783695, + "instruction": "/bin/sh -c wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua && chmod +x /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 368879851, + "status": "stale", + "last_pulled": "2021-01-12T17:24:17.88938Z", + "last_pushed": "2020-10-15T14:32:37.681063Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags/latest.info.json new file mode 100644 index 0000000000..4b1f4dc39d --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/bitbucket-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 105714713, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:0205a9dfcd1de546e61e54216a0ea20b549831e605f2207d9fd25976dbe8914e", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368879851, + "status": "stale", + "last_pulled": "2021-01-12T17:24:17.88938Z", + "last_pushed": "2020-10-15T14:32:37.681063Z" + } + ], + "last_updated": "2020-10-15T14:32:37.681063Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9299518, + "full_size": 368879851, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-12T17:24:17.88938Z", + "tag_last_pushed": "2020-10-15T14:32:37.681063Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/Dockerfile new file mode 100644 index 0000000000..9b78304ce6 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/info.json new file mode 100644 index 0000000000..5c93942549 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "clutch-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11078089, + "last_updated": "2020-08-20T14:53:12.542994Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "clutch-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Clutch](http://archiveteam.org/index.php?title=Clutch)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Clutch\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/clutch-grab.git; cd clutch-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/clutch-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, clutch-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/clutch-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/clutch-grab\n cd clutch-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #pearls.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/sources.json new file mode 100644 index 0000000000..d2e21861bc --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/a199da12-3f64-4fb0-b8de-e6b53807afbe/" + ], + "channel": "Stable", + "image": "warcforceone/clutch-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "clutch-grab", + "resource_uri": "/api/build/v1/source/4e33a9d4-d56f-432e-b0d0-79591a919f20/", + "state": "Failed", + "uuid": "4e33a9d4-d56f-432e-b0d0-79591a919f20" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags.json new file mode 100644 index 0000000000..15f79e5f2a --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 112683426, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:4e84f56d35afe3c9c98e579b5e21d4d8eaed315a1b4f961d2fc85476d8ed56ec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368321345, + "status": "active", + "last_pulled": "2021-02-09T18:14:02.880747Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-20T14:53:12.187023Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9612088, + "full_size": 368321345, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T18:14:02.880747Z", + "tag_last_pushed": "2020-08-20T14:53:12.187023Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags/latest.images.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags/latest.info.json new file mode 100644 index 0000000000..7f3dd130ef --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/clutch-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 112683426, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:4e84f56d35afe3c9c98e579b5e21d4d8eaed315a1b4f961d2fc85476d8ed56ec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368321345, + "status": "active", + "last_pulled": "2021-02-09T18:14:02.880747Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-20T14:53:12.187023Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9612088, + "full_size": 368321345, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T18:14:02.880747Z", + "tag_last_pushed": "2020-08-20T14:53:12.187023Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/Dockerfile new file mode 100644 index 0000000000..9f3938345b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:edge +RUN apk upgrade --no-cache \ + && apk add --no-cache dnsmasq +COPY dnsmasq.conf /etc/dnsmasq.conf +STOPSIGNAL SIGKILL +ENTRYPOINT ["dnsmasq", "-kC/etc/dnsmasq.conf"] diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/info.json new file mode 100644 index 0000000000..38fa1ffeda --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "dnsmasq", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 21054150, + "last_updated": "2021-02-20T00:18:15.003220Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/sources.json new file mode 100644 index 0000000000..db98452087 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/5a9868ba-14b5-41f4-b5ec-63fb2d1fa8a5/" + ], + "channel": "Stable", + "image": "warcforceone/dnsmasq", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "dnsmasq-df", + "resource_uri": "/api/build/v1/source/f7fc079c-ba63-4734-a23e-1d1f090fcd5d/", + "state": "Success", + "uuid": "f7fc079c-ba63-4734-a23e-1d1f090fcd5d" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags.json new file mode 100644 index 0000000000..d870f1757b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 103588681, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:18d8db27db7e148e707eb8ee5686fddb32704a9165b35ef438973d14524cd3c5", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 4477304, + "status": "active", + "last_pulled": "2021-02-21T12:24:17.433242Z", + "last_pushed": "2021-02-20T00:18:14.704711Z" + } + ], + "last_updated": "2021-02-20T00:18:14.704711Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9191572, + "full_size": 4477304, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T12:24:17.433242Z", + "tag_last_pushed": "2021-02-20T00:18:14.704711Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags/latest.images.json new file mode 100644 index 0000000000..82d738e2b4 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags/latest.images.json @@ -0,0 +1,44 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:18d8db27db7e148e707eb8ee5686fddb32704a9165b35ef438973d14524cd3c5", + "layers": [ + { + "digest": "sha256:fa704576706331d0bbffe493842fe09e56a6cab3aa869749cf873dc431910583", + "size": 2813556, + "instruction": "ADD file:a5c7d53b28a7eec81df661fdab57f00eca5edb2a1acef7726153777cb37e4bf5 in / " + }, + { + "size": 0, + "instruction": " CMD [\"/bin/sh\"]" + }, + { + "digest": "sha256:85cc12a3c308b6ffd8e9bed72a3fa706df87c27172e83232c796b9c0a74b2895", + "size": 1663458, + "instruction": "/bin/sh -c apk upgrade --no-cache && apk add --no-cache dnsmasq" + }, + { + "digest": "sha256:4f937faedb43a4374fa7785cdaa56c2119a1a24aa2fa340168a99a31f7df91af", + "size": 290, + "instruction": "COPY file:e87e66fc03be3f8487a53c1b432ef3465c6c58f4b8b1570121dee97ba046b2b9 in /etc/dnsmasq.conf " + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGKILL" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"dnsmasq\" \"-kC/etc/dnsmasq.conf\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 4477304, + "status": "active", + "last_pulled": "2021-02-21T12:24:17.433242Z", + "last_pushed": "2021-02-20T00:18:14.704711Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags/latest.info.json new file mode 100644 index 0000000000..739549686c --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/dnsmasq/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 103588681, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:18d8db27db7e148e707eb8ee5686fddb32704a9165b35ef438973d14524cd3c5", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 4477304, + "status": "active", + "last_pulled": "2021-02-21T12:24:17.433242Z", + "last_pushed": "2021-02-20T00:18:14.704711Z" + } + ], + "last_updated": "2021-02-20T00:18:14.704711Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9191572, + "full_size": 4477304, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T12:24:17.433242Z", + "tag_last_pushed": "2021-02-20T00:18:14.704711Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/Dockerfile new file mode 100644 index 0000000000..926ace5f79 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/Dockerfile @@ -0,0 +1,4 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at +RUN sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python*/site-packages/seesaw/tracker.py diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/info.json new file mode 100644 index 0000000000..266382704b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "domains-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 151168, + "last_updated": "2021-01-09T14:53:49.620078Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "domains-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [domains](http://archiveteam.org/index.php?title=domains)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"domains\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests warcio\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/domains-grab.git; cd domains-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/domains-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, domains-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/domains-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/domains-grab\n cd domains-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.hackint.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC [#archiveteam-bs](https://webirc.hackint.org/#irc://irc.hackint.org/#archiveteam-bs).\n\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/sources.json new file mode 100644 index 0000000000..8ec658856f --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": false, + "build_settings": [ + "/api/build/v1/setting/9120e450-9874-441f-be56-bd7a56da2477/" + ], + "channel": "Stable", + "image": "warcforceone/domains-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "domains-grab", + "resource_uri": "/api/build/v1/source/a54eca2b-941a-4842-98b2-8ebca1c88850/", + "state": "Success", + "uuid": "a54eca2b-941a-4842-98b2-8ebca1c88850" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags.json new file mode 100644 index 0000000000..e5d22712f1 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 131529596, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:7636ab505a2f7313288afe1f3f362743fe7273d87e2d6e9432e6af92dcc70290", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355240806, + "status": "active", + "last_pulled": "2021-02-20T02:22:04.585308Z", + "last_pushed": "2021-01-09T14:53:43.504044Z" + } + ], + "last_updated": "2021-01-09T14:53:43.504044Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 10571662, + "full_size": 355240806, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:04.585308Z", + "tag_last_pushed": "2021-01-09T14:53:43.504044Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags/latest.images.json new file mode 100644 index 0000000000..9dc83c99c5 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags/latest.images.json @@ -0,0 +1,159 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:7636ab505a2f7313288afe1f3f362743fe7273d87e2d6e9432e6af92dcc70290", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:b401f610667fcd2db4f2c208848d0cf92f2ad490514b5443b3939a45d8b64b4d", + "size": 40895, + "instruction": "COPY dir:42f701425aa6570b0795acaecde911804d3f1da4686513072e0caba2726eeef2 in /grab " + }, + { + "digest": "sha256:59a0759a68a7e6a31ea6fc1f6d559162a678ad5ed66e38b4fe4c315b79ad409b", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + }, + { + "digest": "sha256:27395fd4e260c6ddae6954c3d0ebb1f0e3bd2ee6c3577d949efdaae714ed892b", + "size": 2894, + "instruction": "/bin/sh -c sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python*/site-packages/seesaw/tracker.py" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355240806, + "status": "active", + "last_pulled": "2021-02-20T02:22:04.585308Z", + "last_pushed": "2021-01-09T14:53:43.504044Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags/latest.info.json new file mode 100644 index 0000000000..b9265a0576 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/domains-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 131529596, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:7636ab505a2f7313288afe1f3f362743fe7273d87e2d6e9432e6af92dcc70290", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355240806, + "status": "active", + "last_pulled": "2021-02-20T02:22:04.585308Z", + "last_pushed": "2021-01-09T14:53:43.504044Z" + } + ], + "last_updated": "2021-01-09T14:53:43.504044Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 10571662, + "full_size": 355240806, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:04.585308Z", + "tag_last_pushed": "2021-01-09T14:53:43.504044Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/info.json new file mode 100644 index 0000000000..84a0e29162 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "fotoalbum-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 835123, + "last_updated": "2020-10-31T11:11:27.569458Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "fotoalbum-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Fotoalbum](http://archiveteam.org/index.php?title=Fotoalbum)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Fotoalbum\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/fotoalbum-grab.git; cd fotoalbum-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/fotoalbum-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, fotoalbum-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/fotoalbum-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/fotoalbum-grab\n cd fotoalbum-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #lookatthisfotograph.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/sources.json new file mode 100644 index 0000000000..00142a1706 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": false, + "build_settings": [ + "/api/build/v1/setting/5385947a-fd11-4d67-9bfc-e449ad1c530f/" + ], + "channel": "Stable", + "image": "warcforceone/fotoalbum-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "fotoalbum-grab", + "resource_uri": "/api/build/v1/source/9e8acc0f-7d65-4c36-a9f5-2b2f5678a33f/", + "state": "Success", + "uuid": "9e8acc0f-7d65-4c36-a9f5-2b2f5678a33f" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags.json new file mode 100644 index 0000000000..a5bb54acd8 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 119428392, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:07279a1bdd8c84d98c0db0fbc6b03f7c2819e4d049f1c7bcea1c59e36770ea9b", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356537761, + "status": "active", + "last_pulled": "2021-01-25T09:00:20.61391Z", + "last_pushed": "2020-10-31T11:11:27.176579Z" + } + ], + "last_updated": "2020-10-31T11:11:27.176579Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9933212, + "full_size": 356537761, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-01-25T09:00:20.61391Z", + "tag_last_pushed": "2020-10-31T11:11:27.176579Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags/latest.images.json new file mode 100644 index 0000000000..fee1dff76d --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:07279a1bdd8c84d98c0db0fbc6b03f7c2819e4d049f1c7bcea1c59e36770ea9b", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.0" + }, + { + "digest": "sha256:b1d09d0eabcb6f4e10ef0d360714bc5c780ea017bbcdc1a7e1a4a617ff8fb431", + "size": 20579241, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:475299e7c7f3a160843a7c349c788b8c9ec2ce9dd3b924e42830a38995955c9d", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:d2fe14d8e6bcb0ddb14c3ae2bc50b0f9cad6f42b33935609d160555f3525d71e", + "size": 2120746, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:76cccca5d2434bc713235d52a22280085ecc46e033071abae779166dd01288d7", + "size": 15354788, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:23ae9b84d8c637d7be11896ab80072718fff61a7f7779bafdab94a2e1a944695", + "size": 125, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-10-31T10:57:28Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=0865fe5498c6ef4730c784dd9c6a4f4e1f98a4f3" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:97aac8ae4b3219829025132fbd5add15c9560969fb45221f8d0b0af41e788973", + "size": 40124, + "instruction": "COPY dir:363525162a852e652aebbd7e979aa8cdd809532345a582ab56f91e419baaa320 in /grab " + }, + { + "digest": "sha256:bd702c4a8b5cbc85184adfe8a8c26b0ce775a8fceefe7b442258fcde20059764", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 356537761, + "status": "active", + "last_pulled": "2021-01-25T09:00:20.61391Z", + "last_pushed": "2020-10-31T11:11:27.176579Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags/latest.info.json new file mode 100644 index 0000000000..8c655234b4 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/fotoalbum-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 119428392, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:07279a1bdd8c84d98c0db0fbc6b03f7c2819e4d049f1c7bcea1c59e36770ea9b", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356537761, + "status": "active", + "last_pulled": "2021-01-25T09:00:20.61391Z", + "last_pushed": "2020-10-31T11:11:27.176579Z" + } + ], + "last_updated": "2020-10-31T11:11:27.176579Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9933212, + "full_size": 356537761, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-01-25T09:00:20.61391Z", + "tag_last_pushed": "2020-10-31T11:11:27.176579Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/info.json new file mode 100644 index 0000000000..17b141668d --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "github-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 13726777, + "last_updated": "2021-02-03T13:48:21.932838Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "github-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [GitHub](http://archiveteam.org/index.php?title=GitHub)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"GitHub\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/github-grab.git; cd github-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/github-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, github-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/github-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/github-grab\n cd github-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #gitgud.\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/sources.json new file mode 100644 index 0000000000..40a717223a --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/77062f2a-f999-49ce-b7da-81cddd14bcb9/" + ], + "channel": "Stable", + "image": "warcforceone/github-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "github-grab", + "resource_uri": "/api/build/v1/source/81d70921-952f-4507-97c7-7a3c761ee3c7/", + "state": "Success", + "uuid": "81d70921-952f-4507-97c7-7a3c761ee3c7" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags.json new file mode 100644 index 0000000000..284c9eeca9 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 112047604, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2e3c90259794dee5979c9745ac9c73ef3ea5fb639cc4e745e1bcfc1854a7787a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355285549, + "status": "active", + "last_pulled": "2021-02-21T12:12:10.344872Z", + "last_pushed": "2021-02-03T13:48:21.641084Z" + } + ], + "last_updated": "2021-02-03T13:48:21.641084Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9584009, + "full_size": 355285549, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T12:12:10.344872Z", + "tag_last_pushed": "2021-02-03T13:48:21.641084Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags/latest.images.json new file mode 100644 index 0000000000..e2387971a3 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:2e3c90259794dee5979c9745ac9c73ef3ea5fb639cc4e745e1bcfc1854a7787a", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:b25abe0ecc0925ce877cb7b359bdc9edd6e32fc8dde1adcc072a8c4883a15421", + "size": 88533, + "instruction": "COPY dir:84ec40390e24b81471934710a181eca724e76575a723f9297b3c39a1275afc17 in /grab " + }, + { + "digest": "sha256:dbdfcbfa61388a66f82bfefecec90f757fee49684cf94bc47bcbacb4e113f6c2", + "size": 142, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355285549, + "status": "active", + "last_pulled": "2021-02-21T12:12:10.344872Z", + "last_pushed": "2021-02-03T13:48:21.641084Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags/latest.info.json new file mode 100644 index 0000000000..77572dfdc2 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/github-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 112047604, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2e3c90259794dee5979c9745ac9c73ef3ea5fb639cc4e745e1bcfc1854a7787a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355285549, + "status": "active", + "last_pulled": "2021-02-21T12:12:10.344872Z", + "last_pushed": "2021-02-03T13:48:21.641084Z" + } + ], + "last_updated": "2021-02-03T13:48:21.641084Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9584009, + "full_size": 355285549, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T12:12:10.344872Z", + "tag_last_pushed": "2021-02-03T13:48:21.641084Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/info.json new file mode 100644 index 0000000000..9d26d7fe8b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "google-sites-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7181220, + "last_updated": "2021-02-03T14:12:19.864923Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "google-sites-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Google Sites](http://archiveteam.org/index.php?title=Google_Sites)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Google Sites\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/google-sites-grab.git; cd google-sites-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/google-sites-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, google-sites-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/google-sites-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/google-sites-grab\n cd google-sites-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint channel #nearlylostmygoogles.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/sources.json new file mode 100644 index 0000000000..48c3966b5b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/9a200b7e-6ed7-4e5a-8048-094c5fb2458c/" + ], + "channel": "Stable", + "image": "warcforceone/google-sites-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "google-sites-grab", + "resource_uri": "/api/build/v1/source/18835edd-03c6-4e7e-ad53-61fca2dd7c8d/", + "state": "Success", + "uuid": "18835edd-03c6-4e7e-ad53-61fca2dd7c8d" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags.json new file mode 100644 index 0000000000..a047c47d2e --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 120283946, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:ca02cf2b332f49cc3dc48c4795fbf9ea0f0ada8181e8a1f2ad3879101890b276", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355260303, + "status": "active", + "last_pulled": "2021-02-20T17:33:16.397157Z", + "last_pushed": "2021-02-03T14:12:19.517114Z" + } + ], + "last_updated": "2021-02-03T14:12:19.517114Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9974747, + "full_size": 355260303, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T17:33:16.397157Z", + "tag_last_pushed": "2021-02-03T14:12:19.517114Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags/latest.images.json new file mode 100644 index 0000000000..7d0f430af3 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:ca02cf2b332f49cc3dc48c4795fbf9ea0f0ada8181e8a1f2ad3879101890b276", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:099b59f5dd443f225ff2ccf9f8fd5c62b1196ed3b76d4371c51228be4403b223", + "size": 63286, + "instruction": "COPY dir:056e2a67952ab9f3bad6a329f21ab48ab632b7b007755049a4c9e507abe9b3d1 in /grab " + }, + { + "digest": "sha256:acf1a14524374193f5d8b9e87eaafa8e56994dda04ce72a914147c67ca0a54b1", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355260303, + "status": "active", + "last_pulled": "2021-02-20T17:33:16.397157Z", + "last_pushed": "2021-02-03T14:12:19.517114Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags/latest.info.json new file mode 100644 index 0000000000..07f0fbf2bc --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/google-sites-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 120283946, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:ca02cf2b332f49cc3dc48c4795fbf9ea0f0ada8181e8a1f2ad3879101890b276", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355260303, + "status": "active", + "last_pulled": "2021-02-20T17:33:16.397157Z", + "last_pushed": "2021-02-03T14:12:19.517114Z" + } + ], + "last_updated": "2021-02-03T14:12:19.517114Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9974747, + "full_size": 355260303, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T17:33:16.397157Z", + "tag_last_pushed": "2021-02-03T14:12:19.517114Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/Dockerfile new file mode 100644 index 0000000000..5691e6858a --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/Dockerfile @@ -0,0 +1,15 @@ +ARG TLSTYPE=openssl +FROM atdr.meo.ws/archiveteam/wget-lua:v1.20.3-at-${TLSTYPE} AS wget +FROM python:3-slim +COPY --from=wget /wget /usr/local/bin/wget-lua +ENV LC_ALL=C +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket libpsl5 git \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install requests seesaw warcio zstandard \ + && chmod +x /usr/local/bin/wget-lua \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /grab +STOPSIGNAL SIGINT +ENTRYPOINT ["run-pipeline3", "--disable-web-server", "pipeline.py"] diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/info.json new file mode 100644 index 0000000000..480a4da221 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "grab-base", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 50530, + "last_updated": "2021-02-13T11:25:52.346081Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "# grab-base-df\nBase Dockerfile for warrior project grab scripts\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/sources.json new file mode 100644 index 0000000000..ee8a0328b2 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/568fc7d1-559a-40a5-a27f-0706214a48b1/" + ], + "channel": "Stable", + "image": "warcforceone/grab-base", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "grab-base-df", + "resource_uri": "/api/build/v1/source/97b9831e-a2db-4298-ab9d-dabf34a171b3/", + "state": "Success", + "uuid": "97b9831e-a2db-4298-ab9d-dabf34a171b3" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags.json new file mode 100644 index 0000000000..a9b072eadc --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 63789861, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:269f12c7c0d41372eeab2b2a37e19f8dbf26a20328d1bd943b5be0b260a790a1", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 85903786, + "status": "active", + "last_pulled": "2021-02-20T02:21:58.476745Z", + "last_pushed": "2021-02-13T11:25:52.045638Z" + } + ], + "last_updated": "2021-02-13T11:25:52.045638Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470894, + "full_size": 85903786, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:58.476745Z", + "tag_last_pushed": "2021-02-13T11:25:52.045638Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags/latest.images.json new file mode 100644 index 0000000000..8938982bd8 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags/latest.images.json @@ -0,0 +1,105 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:269f12c7c0d41372eeab2b2a37e19f8dbf26a20328d1bd943b5be0b260a790a1", + "layers": [ + { + "digest": "sha256:45b42c59be334ecda0daaa139b2f7d310e45c564c5f12263b1b8e68ec9e810ed", + "size": 27095142, + "instruction": "ADD file:d5c41bfaf15180481d8606f50799297e3f49b8a258c7c2cd988ab2bf0013272d in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:8cd3485318dba8e325f6b665105e10c51f4011cf2e2b4378ae2b92e104ff3566", + "size": 2769103, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tnetbase \t\ttzdata \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:b430e1981978aa8083832db9dc3f927e4336fbccf20f27a0419a5d31f78205b7", + "size": 10888464, + "instruction": "/bin/sh -c set -ex \t\t&& savedAptMark=\"$(apt-mark showmanual)\" \t&& apt-get update && apt-get install -y --no-install-recommends \t\tdpkg-dev \t\tgcc \t\tlibbluetooth-dev \t\tlibbz2-dev \t\tlibc6-dev \t\tlibexpat1-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tliblzma-dev \t\tlibncursesw5-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tmake \t\ttk-dev \t\tuuid-dev \t\twget \t\txz-utils \t\tzlib1g-dev \t\t$(command -v gpg > /dev/null || echo 'gnupg dirmngr') \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tLDFLAGS=\"-Wl,--strip-all\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& apt-mark auto '.*' > /dev/null \t&& apt-mark manual $savedAptMark \t&& find /usr/local -type f -executable -not \\( -name '*tkinter*' \\) -exec ldd '{}' ';' \t\t| awk '/=>/ { print $(NF-1) }' \t\t| sort -u \t\t| xargs -r dpkg-query --search \t\t| cut -d: -f1 \t\t| sort -u \t\t| xargs -r apt-mark manual \t&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \t&& rm -rf /var/lib/apt/lists/* \t\t&& python3 --version" + }, + { + "digest": "sha256:37d877dc41c386154662483278f5b856d8c34e200d571eb41cee562506fa5bfb", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=21.0.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/4be3fe44ad9dedc028629ed1497052d65d281b8e/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=8006625804f55e1bd99ad4214fd07082fee27a1c35945648a58f9087a714e9d4" + }, + { + "digest": "sha256:0b70d4ba14ef92ccaa7839657e6fca3422f09041c50a44011d6a2382d0c23fe8", + "size": 2452248, + "instruction": "/bin/sh -c set -ex; \t\tsavedAptMark=\"$(apt-mark showmanual)\"; \tapt-get update; \tapt-get install -y --no-install-recommends wget; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tapt-mark auto '.*' > /dev/null; \t[ -z \"$savedAptMark\" ] || apt-mark manual $savedAptMark; \tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \trm -rf /var/lib/apt/lists/*; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "digest": "sha256:2a5987f3c885013cfbe6ee2d5fa8bf80cbc61323e0ac8aa63ab2720faa2b9531", + "size": 926625, + "instruction": "COPY file:c7a381cd8311209dda148a7d15ef252c496da848907afaa18f8371d09648021e in /usr/local/bin/wget-lua " + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:4c4e90548af1cdbeb6d8ce5cde1c907ac0a4e48aed698d9a80b2afd3d3128111", + "size": 41771879, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket libpsl5 git && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0c0bb03728b9b8735afce4e3653c7761db07f3245cfa66a44bda45f8022184c8", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 85903786, + "status": "active", + "last_pulled": "2021-02-20T02:21:58.476745Z", + "last_pushed": "2021-02-13T11:25:52.045638Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags/latest.info.json new file mode 100644 index 0000000000..d732c8707a --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/grab-base/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 63789861, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:269f12c7c0d41372eeab2b2a37e19f8dbf26a20328d1bd943b5be0b260a790a1", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 85903786, + "status": "active", + "last_pulled": "2021-02-20T02:21:58.476745Z", + "last_pushed": "2021-02-13T11:25:52.045638Z" + } + ], + "last_updated": "2021-02-13T11:25:52.045638Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470894, + "full_size": 85903786, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:58.476745Z", + "tag_last_pushed": "2021-02-13T11:25:52.045638Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/Dockerfile new file mode 100644 index 0000000000..d12e82a3e1 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:stretch-slim +RUN echo 'deb http://ftp.de.debian.org/debian buster-backports main' >> /etc/apt/sources.list +RUN DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install python python2.7 rsync git ca-certificates curl python-pip \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard && pip install requests +COPY * factory/ +RUN rm -rf /factory/megawarc && git clone https://github.com/archiveteam/megawarc.git /factory/megawarc +WORKDIR /factory +COPY docker-boot.sh / +RUN chmod +x /docker-boot.sh +ENTRYPOINT ["/docker-boot.sh"] diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/info.json new file mode 100644 index 0000000000..ed91904f57 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "megawarc-factory", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 80, + "last_updated": "2020-12-29T13:44:29.663201Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "Archive Team megawarc factory\n=============================\nSome scripts to bundle Archive Team uploads and upload them to Archive.org. Use at your own risk; the scripts will need per-project adjustment.\n\nThese scripts make batches of uploaded warc.gz files, combine them into megawarcs and upload them to their permanent home on Archive.org.\n\nThree processes work together to make this happen:\n\n1. The chunker\n--------------\nThe chunker moves uploaded warc.gz files from the upload directory to a batch directory. When this directory has grown to 50GB, the chunker begins a new directory and moves the completed directory to the packing queue.\n\nThere can only be one chunker per upload directory. Chunking doesn't take long, if the files are not moving to a different filesystem.\n\n2. The packer\n-------------\nThe packer monitors the packing queue. When the chunker brings a new directory, the packer removes the directory from the queue and starts converting it into a megawarc (using the megawarc utility). When that is done, the packer moves the megawarc to the upload queue and removes the original warc files.\n\nIf necessary, multiple packers can work the same queue. Packing involves lots of gzipping and takes some time.\n\n3. The uploader\n---------------\nThe uploader monitors the upload queue. When the packer brings a new megawarc, the uploader removes the megawarc from the queue and uploads it to Archive.org. If the upload is successful, the uploader removes the megawarc.\n\nIf necessary, multiple uploaders can work the same queue.\n\n4. The offloader\n---------------\nThe offloader monitors the upload queue. Instead of uploading to Archive.org, the megawarc will be sent to another host via rsync. This is useful when Archive.org has issues. \n\nThis can be used at the same time as the uploader without issues.\n\n\nFilesystems\n-----------\nFrom the chunker to the uploader, the chunks move through the system as timestamped directories, e.g., 20130401213900.) This timestamp will also be used in the name of the uploaded item on Archive.org. The queues are directories. Processes 'claim' a chunk by moving it from the queue directory to their working directory. This assumes that `mv` is an atomic operation.\n\nFor efficiency and to maintain the atomicity of `mv`, the filesystem of the directories is very important:\n\n1. The Rsync upload directory, the chunker working directory, the packing queue and that side of the packer's working directory should all be on the same filesystem. This ensures that the uploaded warc.gz files never move to a different file system.\n2. The megawarc side of the packer's working directory, the upload queue and the uploader's working directory should also share a filesystem.\n\nFilesystems 1 and 2 do not have to be the same.\n\n\nConfiguration\n-------------\nCreate a configuration file called `config.sh` and place it in the directory where you start the scripts. See the `config.example.sh` for more details.\n\n\nRunning\n-------\nRun the scripts in `screen`, `tmux` or something similar. `touch RUN` before you start the scripts. Use `rm RUN` to stop gracefully.\n\n* `./chunk-multiple` (run exactly one)\n* `./pack-multiple` (you may run more than one)\n* `./upload-multiple` (you may run more than one)\n* `./offload-multiple` (you may run more than one, can work in tandem with `upload-multiple`)\n\nUtility scripts:\n\n* `./du-all` will run `du -hs` in all queues\n\n\nScheduling priorities\n---------------------\nThe packing script will use all your I/O capacity. Consider using `nice` and `ionice` to run in at a lower priority, so it doesn't hinder your incoming Rsync or outgoing curl uploads.\n\n* `ionice -c 2 -n 6 nice -n 19 ./pack-multiple`\n\n\nRecovering from errors\n----------------------\nThe scripts are designed not to lose data. If a script dies, you can look in its working directory for in-progress items and move them back to the queue.\n\n\nRequirements\n------------\nThese scripts use Bash and Curl.\n\nYou should clone https://github.com/ArchiveTeam/megawarc to the `megawarc/` subdirectory of these scripts. The megawarc utility requires Python and Gzip.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/sources.json new file mode 100644 index 0000000000..a3169843fa --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/dc5abaaa-13e5-4b5a-952b-cf2d5054b6cb/" + ], + "channel": "Stable", + "image": "warcforceone/megawarc-factory", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "archiveteam-megawarc-factory", + "resource_uri": "/api/build/v1/source/42f121c3-bd58-4f01-bcd6-1563dac5cf4a/", + "state": "Success", + "uuid": "42f121c3-bd58-4f01-bcd6-1563dac5cf4a" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags.json new file mode 100644 index 0000000000..5cb4b4853c --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 1202997, + "id": 62496302, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:34a7d52683683018fd803c10087e27b6f2b5a4c499bcf4bf600983b8f38e3f51", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 83378492, + "status": "stale", + "last_pulled": "2021-01-19T17:27:32.510273Z", + "last_pushed": "2020-12-29T13:44:29.105803Z" + } + ], + "last_updated": "2020-12-29T13:44:29.105803Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7415529, + "full_size": 83378492, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-19T17:27:32.510273Z", + "tag_last_pushed": "2020-12-29T13:44:29.105803Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags/latest.images.json new file mode 100644 index 0000000000..a5133a46a1 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags/latest.images.json @@ -0,0 +1,64 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:34a7d52683683018fd803c10087e27b6f2b5a4c499bcf4bf600983b8f38e3f51", + "layers": [ + { + "digest": "sha256:e50c3c9ef5a201a24959788dcbc7ebf88d95c63e132a4d7396ce541127afd88e", + "size": 22527860, + "instruction": "ADD file:f03e68a10b84e2342cfffbb8cdec1117c7f5e5d0dd004072e84efb62cfdf157c in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:7835e0e8e88fdf51a188a95e2e5568fa8f5312cb64f0876caf424ee3a27f2049", + "size": 296, + "instruction": "/bin/sh -c echo 'deb http://ftp.de.debian.org/debian buster-backports main' >> /etc/apt/sources.list" + }, + { + "digest": "sha256:05082f7965abaec3bff308fbc6f539d4b720825f4370ac45de47457b588e4d35", + "size": 60447857, + "instruction": "/bin/sh -c DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install python python2.7 rsync git ca-certificates curl python-pip && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard && pip install requests" + }, + { + "digest": "sha256:77a7f73d7caba36c241f1baeb5a8cb5e42da767b1dad8e4d1073ff550165e3f0", + "size": 233941, + "instruction": "COPY multi:1c3cbf1e872449e49263d6a6ca9dc6d0aeac6034ec7b51cedcee86a674988d7d in factory/ " + }, + { + "digest": "sha256:bee00402a192e2d99dec3b0a9cf03a757fd0d2244d7ba265fff8ca1862ebcf26", + "size": 166800, + "instruction": "/bin/sh -c rm -rf /factory/megawarc && git clone https://github.com/archiveteam/megawarc.git /factory/megawarc" + }, + { + "size": 0, + "instruction": "WORKDIR /factory" + }, + { + "digest": "sha256:391913c444fc4ede6574dfec5fc56b465d5647609cd093282f339c32a8a3f126", + "size": 869, + "instruction": "COPY file:3b1d6271f1ce8e2709239da0ccf7e9ef0619e35964033fb297cac2918ccd8e0f in / " + }, + { + "digest": "sha256:391913c444fc4ede6574dfec5fc56b465d5647609cd093282f339c32a8a3f126", + "size": 869, + "instruction": "/bin/sh -c chmod +x /docker-boot.sh" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"/docker-boot.sh\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 83378492, + "status": "stale", + "last_pulled": "2021-01-19T17:27:32.510273Z", + "last_pushed": "2020-12-29T13:44:29.105803Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags/latest.info.json new file mode 100644 index 0000000000..c55c299454 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/megawarc-factory/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 1202997, + "id": 62496302, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:34a7d52683683018fd803c10087e27b6f2b5a4c499bcf4bf600983b8f38e3f51", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 83378492, + "status": "stale", + "last_pulled": "2021-01-19T17:27:32.510273Z", + "last_pushed": "2020-12-29T13:44:29.105803Z" + } + ], + "last_updated": "2020-12-29T13:44:29.105803Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7415529, + "full_size": 83378492, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-19T17:27:32.510273Z", + "tag_last_pushed": "2020-12-29T13:44:29.105803Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/Dockerfile new file mode 100644 index 0000000000..8ca159063d --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket luarocks \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +RUN luarocks install cgilua +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/info.json new file mode 100644 index 0000000000..76cfd0280c --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "mercurial-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 1139185, + "last_updated": "2020-10-31T10:57:10.413100Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "mercurial-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Mercurial](http://archiveteam.org/index.php?title=Mercurial)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Mercurial\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync luarocks \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n pip install --upgrade seesaw zstandard\n luarocks install cgilua\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/mercurial-grab.git; cd mercurial-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/mercurial-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync luarocks \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For Fedora:\n\n dnf -y groupinstall \"Development Tools\"\n dnf -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, mercurial-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/mercurial-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/mercurial-grab\n cd mercurial-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/sources.json new file mode 100644 index 0000000000..6fb6b33c0e --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/a01bc9a2-31eb-4898-9e91-5b00aaf3422d/" + ], + "channel": "Stable", + "image": "warcforceone/mercurial-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "mercurial-grab", + "resource_uri": "/api/build/v1/source/bebe2412-9622-49d4-947e-d05034aac71a/", + "state": "Success", + "uuid": "bebe2412-9622-49d4-947e-d05034aac71a" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags.json new file mode 100644 index 0000000000..b380f76cc8 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 104901174, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:f2780378a7690b2371048e4eab8c1b636f59b50790a8280c87182c34dd289d3d", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369671957, + "status": "stale", + "last_pulled": "2020-11-13T00:16:22.577072Z", + "last_pushed": "2020-10-31T10:57:10.072246Z" + } + ], + "last_updated": "2020-10-31T10:57:10.072246Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9259478, + "full_size": 369671957, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-13T00:16:22.577072Z", + "tag_last_pushed": "2020-10-31T10:57:10.072246Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags/latest.images.json new file mode 100644 index 0000000000..fa2cf5e0e8 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags/latest.images.json @@ -0,0 +1,164 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:f2780378a7690b2371048e4eab8c1b636f59b50790a8280c87182c34dd289d3d", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:7f7c8879a275c860b98da629884e116f34e7531db5595b6d8c22653883506ed2", + "size": 2120572, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:4734814b4cdffab76ba2d4228048572e21d13069ad52a46881c48b0789026a79", + "size": 15176911, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:febbd2aa869b044d7f8a9fb316bfadde45653860dee26ec0473aebbe642c4535", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-10-31T00:11:54Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=474a18cb229913cca0bea8309fd791a1c8975512" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:6ab65b315a4128b4566967d14104bd3b750be539a6089355878964e41e0b7041", + "size": 15459068, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket luarocks && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:4ca2a36a88f3b03f66d1669bcd28e1c73e26b793e4b9764e9d32b737980d6da8", + "size": 224053, + "instruction": "/bin/sh -c luarocks install cgilua" + }, + { + "digest": "sha256:760a6e1318bb7e86a834208ffd999e52c6f34864a9bc1786b076ce868094e116", + "size": 59572, + "instruction": "COPY dir:1a8346118c0d2a503aa85164193700c53361dfbe743bce35d9dd4928719f6b72 in /grab " + }, + { + "digest": "sha256:576f93ae126c14bf9c0440782d51877cd603e1ce81bede111ac0ea584ddd6b94", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 369671957, + "status": "stale", + "last_pulled": "2020-11-13T00:16:22.577072Z", + "last_pushed": "2020-10-31T10:57:10.072246Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags/latest.info.json new file mode 100644 index 0000000000..97a848b72c --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/mercurial-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 104901174, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:f2780378a7690b2371048e4eab8c1b636f59b50790a8280c87182c34dd289d3d", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369671957, + "status": "stale", + "last_pulled": "2020-11-13T00:16:22.577072Z", + "last_pushed": "2020-10-31T10:57:10.072246Z" + } + ], + "last_updated": "2020-10-31T10:57:10.072246Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9259478, + "full_size": 369671957, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-13T00:16:22.577072Z", + "tag_last_pushed": "2020-10-31T10:57:10.072246Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/Dockerfile new file mode 100644 index 0000000000..9b78304ce6 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/info.json new file mode 100644 index 0000000000..4605a17d1d --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "naver-matome-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 8478031, + "last_updated": "2020-09-22T15:42:01.244574Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "naver-matome-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [NAVER\u307e\u3068\u3081](http://archiveteam.org/index.php?title=NAVER\u307e\u3068\u3081)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"NAVER\u307e\u3068\u3081\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/naver-matome-grab.git; cd naver-matome-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/naver-matome-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, naver-matome-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/naver-matome-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/naver-matome-grab\n cd naver-matome-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #navergonnagiveyouup.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/sources.json new file mode 100644 index 0000000000..3e63508462 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": false, + "build_settings": [ + "/api/build/v1/setting/5140fd44-a466-4220-8478-c38ea663b6d1/" + ], + "channel": "Stable", + "image": "warcforceone/naver-matome-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "naver-matome-grab", + "resource_uri": "/api/build/v1/source/f502765c-a143-4a78-8f59-8f85c6c7ae18/", + "state": "Success", + "uuid": "f502765c-a143-4a78-8f59-8f85c6c7ae18" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags.json new file mode 100644 index 0000000000..13e6b4521a --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 118429736, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2bccc575ead8a3987dca7d692ee8bcc43c68b39dd5373d025affd70e91a7be44", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353764508, + "status": "active", + "last_pulled": "2021-02-09T20:45:19.121077Z", + "last_pushed": "2020-09-22T15:42:00.746382Z" + } + ], + "last_updated": "2020-09-22T15:42:00.746382Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9883515, + "full_size": 353764508, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T20:45:19.121077Z", + "tag_last_pushed": "2020-09-22T15:42:00.746382Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags/latest.images.json new file mode 100644 index 0000000000..dca12f21b9 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags/latest.images.json @@ -0,0 +1,138 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:2bccc575ead8a3987dca7d692ee8bcc43c68b39dd5373d025affd70e91a7be44", + "layers": [ + { + "digest": "sha256:57df1a1f1ad841deaf50c8f662d77e93b4b17af776ed66148116607f9aceffa8", + "size": 50395913, + "instruction": "ADD file:07a6578d6f507bd9c51bdf4fe41402db5dcf3b9fdf51cd4315778c27da1add39 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:71e126169501d71bbbd0d3c8d9f35836c41660869fe8432ac606341ed21f7adb", + "size": 7811567, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:1af28a55c3f320826db8df3146a2c198f9042877ef679f9e32210aa9a7fac9ef", + "size": 9996317, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:03f1c9932170e54fface2382b2550b8052ae3d41f27e66ea1294e2055dd2b2e7", + "size": 51829661, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:65b3db15f518f11e53c95664d0675a5d78a5329d18d5316a406c2a45907a0723", + "size": 192249513, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:3e3b8947ed83f2a39a813e0a9172cfdd571d4c943228a74e833af60e7536cac5", + "size": 6145377, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.5" + }, + { + "digest": "sha256:a4850b8bdbb7b624a1aaf5892b7409d5ae64f8dfdbb19e3017cf6e7fbbfb0ef9", + "size": 17998369, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:41653399496823f7855fe3bf2725fe0a1b79f2da59db9f3b8d5d9a5177c896ae", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/fa7dc83944936bf09a0e4cb5d5ec852c0d256599/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6e0bb0a2c2533361d7f297ed547237caf1b7507f197835974c0dd7eba998c53c" + }, + { + "digest": "sha256:1b580f9ce4cedf9f911da821c8445cd807bc29a7ab3cc62937b87604138dbfa8", + "size": 2119419, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:112d3c4c29c93c57de0b348a39deaadc7723662dc02032b81033038d1df819ec", + "size": 15159654, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:7c80e1fa56ef43f8e012bfb982788afe906a1e5cac2cc87dc307a1c376aa74f3", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:2dbac3ba6deca119b8c63326d2644d26ef1a27846fc42904f5e29214521afce1", + "size": 58249, + "instruction": "COPY dir:e7db9d6e2c7d9455fa5cb8a75918994482dc05b0d99915ccf7ba765a1637991e in /grab " + }, + { + "digest": "sha256:b9ae6b60d44890e1acbdaf9b444c52caf286d0130d829cb083264e2eeb3785fe", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 353764508, + "status": "active", + "last_pulled": "2021-02-09T20:45:19.121077Z", + "last_pushed": "2020-09-22T15:42:00.746382Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags/latest.info.json new file mode 100644 index 0000000000..f998b4ac54 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/naver-matome-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 118429736, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2bccc575ead8a3987dca7d692ee8bcc43c68b39dd5373d025affd70e91a7be44", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353764508, + "status": "active", + "last_pulled": "2021-02-09T20:45:19.121077Z", + "last_pushed": "2020-09-22T15:42:00.746382Z" + } + ], + "last_updated": "2020-09-22T15:42:00.746382Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9883515, + "full_size": 353764508, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T20:45:19.121077Z", + "tag_last_pushed": "2020-09-22T15:42:00.746382Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/info.json new file mode 100644 index 0000000000..ab4e59abde --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "pastebin-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 386443, + "last_updated": "2021-02-03T13:42:13.640186Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "pastebin-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Pastebin](http://archiveteam.org/index.php?title=Pastebin)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Pastebin\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/pastebin-grab.git; cd pastebin-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/pastebin-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, pastebin-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/pastebin-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/pastebin-grab\n cd pastebin-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint #pastalavista.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/sources.json new file mode 100644 index 0000000000..2404f8f2f8 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/6cab4c66-a6bb-4239-9489-360d8cb03471/" + ], + "channel": "Stable", + "image": "warcforceone/pastebin-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "pastebin-grab", + "resource_uri": "/api/build/v1/source/7a5ac6cc-051c-4214-81a7-7eafc73cef7a/", + "state": "Success", + "uuid": "7a5ac6cc-051c-4214-81a7-7eafc73cef7a" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags.json new file mode 100644 index 0000000000..2dd2a4ce87 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 102735030, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:32e764fef061c155c249f947259a76f3e431c8236b4c2bd791d141c8e9fea873", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355261696, + "status": "active", + "last_pulled": "2021-02-20T02:22:02.031822Z", + "last_pushed": "2021-02-03T13:42:13.273216Z" + } + ], + "last_updated": "2021-02-03T13:42:13.273216Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9154879, + "full_size": 355261696, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:02.031822Z", + "tag_last_pushed": "2021-02-03T13:42:13.273216Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags/latest.images.json new file mode 100644 index 0000000000..a8c683ed40 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:32e764fef061c155c249f947259a76f3e431c8236b4c2bd791d141c8e9fea873", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:1536bb0c75b05971d8ca7e4d032f4f022286bef1f45d8158194d67faf1af1c07", + "size": 64679, + "instruction": "COPY dir:370d4a1793d1d393c73d1d4f418581af7db7b5e5e7a3e426ae221c8c5d03e5d1 in /grab " + }, + { + "digest": "sha256:ffaf42b9ffca673f5d75ceea50241c2c3b839d155888eb3c1f5cf62e622d36af", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355261696, + "status": "active", + "last_pulled": "2021-02-20T02:22:02.031822Z", + "last_pushed": "2021-02-03T13:42:13.273216Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags/latest.info.json new file mode 100644 index 0000000000..5d11f3574b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/pastebin-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 102735030, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:32e764fef061c155c249f947259a76f3e431c8236b4c2bd791d141c8e9fea873", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355261696, + "status": "active", + "last_pulled": "2021-02-20T02:22:02.031822Z", + "last_pushed": "2021-02-03T13:42:13.273216Z" + } + ], + "last_updated": "2021-02-03T13:42:13.273216Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9154879, + "full_size": 355261696, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:02.031822Z", + "tag_last_pushed": "2021-02-03T13:42:13.273216Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/Dockerfile new file mode 100644 index 0000000000..4ddc498c00 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/Dockerfile @@ -0,0 +1,8 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/info.json new file mode 100644 index 0000000000..1e9313a13b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "reddit-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 5792737, + "last_updated": "2021-01-14T00:56:43.223341Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "reddit-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Reddit](http://archiveteam.org/index.php?title=Reddit)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Reddit\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/reddit-grab.git; cd reddit-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/reddit-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, reddit-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/reddit-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/reddit-grab\n cd reddit-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC #shreddit.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/sources.json new file mode 100644 index 0000000000..ef66da7c9b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/sources.json @@ -0,0 +1,16 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [], + "channel": "Stable", + "image": "warcforceone/reddit-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "reddit-grab", + "resource_uri": "/api/build/v1/source/dc7da00e-9940-41cc-adec-5993f2903fd1/", + "state": "Empty", + "uuid": "dc7da00e-9940-41cc-adec-5993f2903fd1" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags.json new file mode 100644 index 0000000000..b23b3d3350 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 110463951, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:861f767c36c60bc249fa1bb2c692c74eef87e3448e4179a7d72e027e0a3be802", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369971934, + "status": "active", + "last_pulled": "2021-02-04T14:13:47.006022Z", + "last_pushed": "2021-01-14T00:56:42.817606Z" + } + ], + "last_updated": "2021-01-14T00:56:42.817606Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9525328, + "full_size": 369971934, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-04T14:13:47.006022Z", + "tag_last_pushed": "2021-01-14T00:56:42.817606Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags/latest.images.json new file mode 100644 index 0000000000..376a253942 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags/latest.images.json @@ -0,0 +1,159 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:861f767c36c60bc249fa1bb2c692c74eef87e3448e4179a7d72e027e0a3be802", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:d5095accb4b2cc664998ce481eeceeb359fe52ffc449d4ea4923ef5b12890ec4", + "size": 14219511, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:56c04c083a6f74407eb93255b4338ab9aa75269bf33e60b44294f95a6a8adc90", + "size": 555407, + "instruction": "COPY dir:e0f48eaf4d219be53af5143432dbe6d5c9fa9dfdadbec36601e685f6a0c4b9cc in /grab " + }, + { + "digest": "sha256:eca96286643ebbbb9a297d8f16290d503c85efb44d98b0f1a82c69d582127403", + "size": 142, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 369971934, + "status": "active", + "last_pulled": "2021-02-04T14:13:47.006022Z", + "last_pushed": "2021-01-14T00:56:42.817606Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags/latest.info.json new file mode 100644 index 0000000000..ca171d7348 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/reddit-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 110463951, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:861f767c36c60bc249fa1bb2c692c74eef87e3448e4179a7d72e027e0a3be802", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369971934, + "status": "active", + "last_pulled": "2021-02-04T14:13:47.006022Z", + "last_pushed": "2021-01-14T00:56:42.817606Z" + } + ], + "last_updated": "2021-01-14T00:56:42.817606Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9525328, + "full_size": 369971934, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-04T14:13:47.006022Z", + "tag_last_pushed": "2021-01-14T00:56:42.817606Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/Dockerfile new file mode 100644 index 0000000000..b2880e3030 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/Dockerfile @@ -0,0 +1,4 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at +RUN sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python3.8/site-packages/seesaw/tracker.py diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/info.json new file mode 100644 index 0000000000..b86bdc38aa --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "tencent-weibo-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 94271733, + "last_updated": "2020-10-31T00:33:22.019703Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "tencent-weibo-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [\u817e\u8baf\u5fae\u535a](http://archiveteam.org/index.php?title=\u817e\u8baf\u5fae\u535a)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"\u817e\u8baf\u5fae\u535a\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/tencent-weibo-grab.git; cd tencent-weibo-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/tencent-weibo-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, tencent-weibo-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/tencent-weibo-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/tencent-weibo-grab\n cd tencent-weibo-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #twocents.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/sources.json new file mode 100644 index 0000000000..dc7aa42966 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/b376fd6b-3c6d-44da-8a44-9d378adf729f/" + ], + "channel": "Stable", + "image": "warcforceone/tencent-weibo-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "tencent-weibo-grab", + "resource_uri": "/api/build/v1/source/bf630818-4cbb-4c63-89be-ee056a848448/", + "state": "Failed", + "uuid": "bf630818-4cbb-4c63-89be-ee056a848448" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags.json new file mode 100644 index 0000000000..8eff7e5db9 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 117793833, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b276f6f881d8b119906c3bed970b5fed871f323e903b26d39c4201f86acb8aec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353998513, + "status": "active", + "last_pulled": "2021-02-21T08:02:12.256158Z", + "last_pushed": "2020-10-31T00:33:13.098519Z" + } + ], + "last_updated": "2020-10-31T00:33:13.098519Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9852521, + "full_size": 353998513, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T08:02:12.256158Z", + "tag_last_pushed": "2020-10-31T00:33:13.098519Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags/latest.images.json new file mode 100644 index 0000000000..622ebacf19 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:b276f6f881d8b119906c3bed970b5fed871f323e903b26d39c4201f86acb8aec", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:7f7c8879a275c860b98da629884e116f34e7531db5595b6d8c22653883506ed2", + "size": 2120572, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:b6c4e663f7617a6ea954dca6864617839abd937eba3eddcc32a145ceb34bd492", + "size": 15176496, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:9428bde242c46afca174985448a1fbdc1da4d1e75b2294486e478f8f56154b83", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:2f95c1eb578f88241406823b2fe4e90170d3adc2c6669cd6d6cf6aa5e6c4caa0", + "size": 66803, + "instruction": "COPY dir:d4b7e6a37f8f01dd3a5174f6dd485f0f802f21159079b701ad7effa263238ee0 in /grab " + }, + { + "digest": "sha256:2b97698c20375c740fbc41903f033f6e1fa764121b7faef338764b0d612ce794", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + }, + { + "digest": "sha256:f5aaa355ef7d77124c27b1017fb29a75a886262b86409e211270cab95aaa3bfb", + "size": 2895, + "instruction": "/bin/sh -c sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python3.8/site-packages/seesaw/tracker.py" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 353998513, + "status": "active", + "last_pulled": "2021-02-21T08:02:12.256158Z", + "last_pushed": "2020-10-31T00:33:13.098519Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags/latest.info.json new file mode 100644 index 0000000000..700a74fde5 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/tencent-weibo-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 117793833, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b276f6f881d8b119906c3bed970b5fed871f323e903b26d39c4201f86acb8aec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353998513, + "status": "active", + "last_pulled": "2021-02-21T08:02:12.256158Z", + "last_pushed": "2020-10-31T00:33:13.098519Z" + } + ], + "last_updated": "2020-10-31T00:33:13.098519Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9852521, + "full_size": 353998513, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-21T08:02:12.256158Z", + "tag_last_pushed": "2020-10-31T00:33:13.098519Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/Dockerfile new file mode 100644 index 0000000000..ec760688d1 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM warcforceone/grab-base +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua \ + && chmod +x /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/info.json new file mode 100644 index 0000000000..c8c05f5068 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "theartistunion-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 139872, + "last_updated": "2020-10-15T14:40:26.351961Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "theartistunion-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [The Artist Union](http://archiveteam.org/index.php?title=The_Artist_Union)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"The Artist Union\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/theartistunion-grab.git; cd theartistunion-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/theartistunion-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, theartistunion-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/theartistunion-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/theartistunion-grab\n cd theartistunion-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/sources.json new file mode 100644 index 0000000000..9c92239cdf --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/d969e438-189e-4462-a681-b5fb487e80e7/" + ], + "channel": "Stable", + "image": "warcforceone/theartistunion-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "theartistunion-grab", + "resource_uri": "/api/build/v1/source/f55a42ba-2bed-4dc0-b98c-5434e2bd9dc6/", + "state": "Failed", + "uuid": "f55a42ba-2bed-4dc0-b98c-5434e2bd9dc6" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags.json new file mode 100644 index 0000000000..76a932ca67 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 111290282, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:6184157bd69442553c4492f8eaa324d3ae36d279fae860cd877defd93d7c0f03", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369356100, + "status": "active", + "last_pulled": "2021-02-20T02:22:08.491635Z", + "last_pushed": "2020-10-15T14:40:25.796092Z" + } + ], + "last_updated": "2020-10-15T14:40:25.796092Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9555565, + "full_size": 369356100, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:08.491635Z", + "tag_last_pushed": "2020-10-15T14:40:25.796092Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags/latest.images.json new file mode 100644 index 0000000000..5dc5f2bfbc --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:6184157bd69442553c4492f8eaa324d3ae36d279fae860cd877defd93d7c0f03", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/fa7dc83944936bf09a0e4cb5d5ec852c0d256599/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6e0bb0a2c2533361d7f297ed547237caf1b7507f197835974c0dd7eba998c53c" + }, + { + "digest": "sha256:3eb2f327eb62905a17b993e9807eeb1c85f9c0172b0721b089586c9fdd4acaee", + "size": 2119480, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:0a47d90c3940a5dd80561f5659e27a5a7f8f7675d95420319ca74039ed94a671", + "size": 15164957, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b538559a01f82dea3b4dfad9ae58884da104ac09290b66f24874efe1a1dd458b", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:78145b7a069bdc3c517a0c6d48ab2d29f6110a1fa7235aab8f7bc597d30c70a2", + "size": 14111453, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:52f9e16adb32d36567a9bd8113b6ebe970f4d6e4b9e8d491dda607e315923e7b", + "size": 544911, + "instruction": "COPY dir:4eeba284110b968df3c1a9b19ac0d92d269c9bf5f574a47c800feb0ae7d60d19 in /grab " + }, + { + "digest": "sha256:a50420f2f9cd1dee61d17b1e20d27a30536fe4f024be74469f0b9588fe67876a", + "size": 783695, + "instruction": "/bin/sh -c wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua && chmod +x /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 369356100, + "status": "active", + "last_pulled": "2021-02-20T02:22:08.491635Z", + "last_pushed": "2020-10-15T14:40:25.796092Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags/latest.info.json new file mode 100644 index 0000000000..191bb71b92 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/theartistunion-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 111290282, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:6184157bd69442553c4492f8eaa324d3ae36d279fae860cd877defd93d7c0f03", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369356100, + "status": "active", + "last_pulled": "2021-02-20T02:22:08.491635Z", + "last_pushed": "2020-10-15T14:40:25.796092Z" + } + ], + "last_updated": "2020-10-15T14:40:25.796092Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9555565, + "full_size": 369356100, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:08.491635Z", + "tag_last_pushed": "2020-10-15T14:40:25.796092Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/Dockerfile b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/info.json new file mode 100644 index 0000000000..d4740cc471 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "youtube-playlistnotes-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11642977, + "last_updated": "2020-10-31T11:04:04.603216Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "youtube-playlistnotes-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [YouTube](http://archiveteam.org/index.php?title=YouTube)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"YouTube Playlist Notes\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/youtube-playlistnotes-grab.git; cd youtube-playlistnotes-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/youtube-playlistnotes-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, youtube-playlistnotes-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/youtube-playlistnotes-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/youtube-playlistnotes-grab\n cd youtube-playlistnotes-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC #down-the-tube.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/sources.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/sources.json new file mode 100644 index 0000000000..4f1b81964b --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/4aed3782-e983-4a07-9fdf-040a1ff5da57/" + ], + "channel": "Stable", + "image": "warcforceone/youtube-playlistnotes-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "youtube-playlistnotes-grab", + "resource_uri": "/api/build/v1/source/e8de687f-0927-4677-a203-abeb641f1442/", + "state": "Success", + "uuid": "e8de687f-0927-4677-a203-abeb641f1442" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags.json new file mode 100644 index 0000000000..a34296ec58 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 119983947, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:9893037e4182af8654f811f2802888178985cc21dcf2e5189a8e7672fbbda3c3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356557380, + "status": "active", + "last_pulled": "2021-02-09T17:42:24.738939Z", + "last_pushed": "2020-10-31T11:04:01.777068Z" + } + ], + "last_updated": "2020-10-31T11:04:01.777068Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9961945, + "full_size": 356557380, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T17:42:24.738939Z", + "tag_last_pushed": "2020-10-31T11:04:01.777068Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.images.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.images.json new file mode 100644 index 0000000000..b348e6fd63 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:9893037e4182af8654f811f2802888178985cc21dcf2e5189a8e7672fbbda3c3", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.0" + }, + { + "digest": "sha256:b1d09d0eabcb6f4e10ef0d360714bc5c780ea017bbcdc1a7e1a4a617ff8fb431", + "size": 20579241, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:475299e7c7f3a160843a7c349c788b8c9ec2ce9dd3b924e42830a38995955c9d", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:d2fe14d8e6bcb0ddb14c3ae2bc50b0f9cad6f42b33935609d160555f3525d71e", + "size": 2120746, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:76cccca5d2434bc713235d52a22280085ecc46e033071abae779166dd01288d7", + "size": 15354788, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:23ae9b84d8c637d7be11896ab80072718fff61a7f7779bafdab94a2e1a944695", + "size": 125, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-10-31T10:57:28Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=0865fe5498c6ef4730c784dd9c6a4f4e1f98a4f3" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:445ee09bf57b640913485a6d84d0ed2a65c5952adb92ecb51d195b9d3bb6c824", + "size": 59743, + "instruction": "COPY dir:7f5f04eeaa8256a4187102a73550e799899740cca02802ec30f8045a31d979cf in /grab " + }, + { + "digest": "sha256:8cc11b70302c8774d19f86a1562f21c90094604b8125b81d4be5b4ba8ecea8be", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 356557380, + "status": "active", + "last_pulled": "2021-02-09T17:42:24.738939Z", + "last_pushed": "2020-10-31T11:04:01.777068Z" + } +] \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.info.json b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.info.json new file mode 100644 index 0000000000..3a2b6f9e83 --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 119983947, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:9893037e4182af8654f811f2802888178985cc21dcf2e5189a8e7672fbbda3c3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356557380, + "status": "active", + "last_pulled": "2021-02-09T17:42:24.738939Z", + "last_pushed": "2020-10-31T11:04:01.777068Z" + } + ], + "last_updated": "2020-10-31T11:04:01.777068Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9961945, + "full_size": 356557380, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T17:42:24.738939Z", + "tag_last_pushed": "2020-10-31T11:04:01.777068Z" +} \ No newline at end of file diff --git a/.tmp.qEgXmoMtcA/warcforceone/user.json b/.tmp.qEgXmoMtcA/warcforceone/user.json new file mode 100644 index 0000000000..3f41e7dc9d --- /dev/null +++ b/.tmp.qEgXmoMtcA/warcforceone/user.json @@ -0,0 +1,12 @@ +{ + "id": "834cf58376f642088956473cac0d4135", + "orgname": "warcforceone", + "full_name": "WARC Force One", + "location": "", + "company": "Archive Team", + "profile_url": "", + "date_joined": "2019-07-06T22:38:01.656830Z", + "gravatar_url": "https://secure.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e.jpg?s=80&r=g&d=mm", + "gravatar_email": "", + "type": "Organization" +} \ No newline at end of file diff --git a/archiveteam/repositories.json b/archiveteam/repositories.json index 5659b5d9d2..540e1dcd81 100644 --- a/archiveteam/repositories.json +++ b/archiveteam/repositories.json @@ -10,7 +10,7 @@ "is_automated": true, "can_edit": false, "star_count": 16, - "pull_count": 10897673, + "pull_count": 10899145, "last_updated": "2018-12-17T00:42:59.374566Z", "is_migrated": true, "collaborator_count": 0, diff --git a/archiveteam/repositories/warrior-dockerfile/info.json b/archiveteam/repositories/warrior-dockerfile/info.json index b72cb4d916..5fdf0bc672 100644 --- a/archiveteam/repositories/warrior-dockerfile/info.json +++ b/archiveteam/repositories/warrior-dockerfile/info.json @@ -9,7 +9,7 @@ "is_automated": true, "can_edit": false, "star_count": 16, - "pull_count": 10897679, + "pull_count": 10899145, "last_updated": "2018-12-17T00:42:59.374566Z", "is_migrated": true, "collaborator_count": 0, diff --git a/archiveteam/repositories/warrior-dockerfile/tags.json b/archiveteam/repositories/warrior-dockerfile/tags.json index 8405a09bd3..b05f07a177 100644 --- a/archiveteam/repositories/warrior-dockerfile/tags.json +++ b/archiveteam/repositories/warrior-dockerfile/tags.json @@ -14,7 +14,7 @@ "os_version": null, "size": 119940149, "status": "active", - "last_pulled": "2021-02-21T11:13:02.635774Z", + "last_pulled": "2021-02-21T13:13:03.009491Z", "last_pushed": null } ], @@ -26,7 +26,7 @@ "full_size": 119940149, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-02-21T11:13:02.635774Z", + "tag_last_pulled": "2021-02-21T13:13:03.009491Z", "tag_last_pushed": "2018-12-17T00:42:59.092382Z" } ] \ No newline at end of file diff --git a/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json b/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json index e50b18b848..76ea27a56f 100644 --- a/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json +++ b/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json @@ -164,7 +164,7 @@ "os_version": null, "size": 119940149, "status": "active", - "last_pulled": "2021-02-21T11:13:02.635774Z", + "last_pulled": "2021-02-21T13:13:03.009491Z", "last_pushed": null } ] \ No newline at end of file diff --git a/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json b/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json index 85590446b2..206df37447 100644 --- a/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json +++ b/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json @@ -13,7 +13,7 @@ "os_version": null, "size": 119940149, "status": "active", - "last_pulled": "2021-02-21T11:13:02.635774Z", + "last_pulled": "2021-02-21T13:13:03.009491Z", "last_pushed": null } ], @@ -25,6 +25,6 @@ "full_size": 119940149, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-02-21T11:13:02.635774Z", + "tag_last_pulled": "2021-02-21T13:13:03.009491Z", "tag_last_pushed": "2018-12-17T00:42:59.092382Z" } \ No newline at end of file diff --git a/dockerhub-metadata.retrieve.log b/dockerhub-metadata.retrieve.log index c9ea7d6bef..4396d36650 100644 --- a/dockerhub-metadata.retrieve.log +++ b/dockerhub-metadata.retrieve.log @@ -1,234 +1,234 @@ -2021-02-21T12:00:01.862686Z Retrieving Docker Hub user archiveteam -2021-02-21T12:00:01.863185Z Fetching https://hub.docker.com/v2/users/archiveteam/ -2021-02-21T12:00:03.376564Z Fetching https://hub.docker.com/v2/repositories/archiveteam/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:04.403484Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/ -2021-02-21T12:00:09.241746Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/dockerfile/ -2021-02-21T12:00:10.318661Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:11.469798Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:12.047906Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/ -2021-02-21T12:00:12.630014Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/images -2021-02-21T12:00:12.672275Z Done with archiveteam -2021-02-21T12:00:12.672860Z Retrieving Docker Hub user warcforceone -2021-02-21T12:00:12.672995Z Fetching https://hub.docker.com/v2/users/warcforceone/ -2021-02-21T12:00:13.501148Z Fetching https://hub.docker.com/v2/repositories/warcforceone/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:14.038916Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/ -2021-02-21T12:00:14.734060Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/dockerfile/ -2021-02-21T12:00:15.321424Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:16.323902Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:16.914919Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/ -2021-02-21T12:00:17.496793Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/images -2021-02-21T12:00:18.076833Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/ -2021-02-21T12:00:18.701638Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/dockerfile/ -2021-02-21T12:00:19.346006Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:20.262603Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:20.852864Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/ -2021-02-21T12:00:21.445343Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/images -2021-02-21T12:00:22.033931Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/ -2021-02-21T12:00:22.654827Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/dockerfile/ -2021-02-21T12:00:23.246619Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:24.194266Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:24.785069Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/ -2021-02-21T12:00:25.391463Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/images -2021-02-21T12:00:25.972467Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/ -2021-02-21T12:00:26.586628Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/dockerfile/ -2021-02-21T12:00:27.175909Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:28.093172Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:28.677042Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/ -2021-02-21T12:00:29.273051Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/images -2021-02-21T12:00:29.852554Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/ -2021-02-21T12:00:30.485858Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/dockerfile/ -2021-02-21T12:00:31.088526Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:32.031894Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:32.622349Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/ -2021-02-21T12:00:33.212568Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/images -2021-02-21T12:00:33.790509Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/ -2021-02-21T12:00:34.475895Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/dockerfile/ -2021-02-21T12:00:35.073814Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:36.077020Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:36.651894Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/ -2021-02-21T12:00:37.239687Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/images -2021-02-21T12:00:37.818616Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/ -2021-02-21T12:00:38.435460Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/dockerfile/ -2021-02-21T12:00:39.019841Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:39.952894Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:40.534641Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/ -2021-02-21T12:00:41.121495Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/images -2021-02-21T12:00:41.702231Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/ -2021-02-21T12:00:42.313664Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/dockerfile/ -2021-02-21T12:00:42.897114Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:43.879900Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:44.469951Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/ -2021-02-21T12:00:45.066224Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/images -2021-02-21T12:00:45.657556Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/ -2021-02-21T12:00:46.278466Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/dockerfile/ -2021-02-21T12:00:46.869452Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:47.830331Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:48.406213Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/ -2021-02-21T12:00:48.989555Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/images -2021-02-21T12:00:49.587895Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/ -2021-02-21T12:00:50.214473Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/dockerfile/ -2021-02-21T12:00:50.814655Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:51.762068Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:52.407874Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/ -2021-02-21T12:00:52.975894Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/images -2021-02-21T12:00:53.555309Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/ -2021-02-21T12:00:54.174843Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/dockerfile/ -2021-02-21T12:00:54.767779Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:55.734816Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:00:56.308452Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/ -2021-02-21T12:00:56.891897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/images -2021-02-21T12:00:57.469025Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/ -2021-02-21T12:00:58.085120Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/dockerfile/ -2021-02-21T12:00:58.697758Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:00:59.647067Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:00.231730Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/ -2021-02-21T12:01:00.823906Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/images -2021-02-21T12:01:01.391230Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/ -2021-02-21T12:01:02.035153Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/dockerfile/ -2021-02-21T12:01:02.616731Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:03.565664Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:04.162689Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/ -2021-02-21T12:01:04.734855Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/images -2021-02-21T12:01:05.326471Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/ -2021-02-21T12:01:05.970931Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/dockerfile/ -2021-02-21T12:01:06.559744Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:07.499559Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:08.075567Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/ -2021-02-21T12:01:08.654368Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/images -2021-02-21T12:01:09.241190Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/ -2021-02-21T12:01:09.861749Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/dockerfile/ -2021-02-21T12:01:10.441019Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:11.377271Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:11.967111Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/ -2021-02-21T12:01:12.554659Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/images -2021-02-21T12:01:13.127507Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/ -2021-02-21T12:01:13.742434Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/dockerfile/ -2021-02-21T12:01:14.323489Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:15.014495Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:15.583900Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/ -2021-02-21T12:01:16.165590Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/images -2021-02-21T12:01:16.742744Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/ -2021-02-21T12:01:17.412862Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/dockerfile/ -2021-02-21T12:01:18.006681Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:18.963179Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:19.551898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/ -2021-02-21T12:01:20.142234Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/images -2021-02-21T12:01:20.749982Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/ -2021-02-21T12:01:21.353957Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/dockerfile/ -2021-02-21T12:01:21.937401Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:22.639612Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:23.215898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/ -2021-02-21T12:01:23.793614Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/images -2021-02-21T12:01:24.378387Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/ -2021-02-21T12:01:25.006669Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/dockerfile/ -2021-02-21T12:01:25.603581Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:26.528753Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:27.095887Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/ -2021-02-21T12:01:27.681502Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/images -2021-02-21T12:01:28.257479Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/ -2021-02-21T12:01:28.893842Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/dockerfile/ -2021-02-21T12:01:29.477627Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:30.407697Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:30.991280Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/ -2021-02-21T12:01:31.569039Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/images -2021-02-21T12:01:32.146814Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/ -2021-02-21T12:01:32.775294Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/dockerfile/ -2021-02-21T12:01:33.375141Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:34.335904Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:34.913264Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/ -2021-02-21T12:01:35.497282Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/images -2021-02-21T12:01:36.073280Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/ -2021-02-21T12:01:36.679387Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/dockerfile/ -2021-02-21T12:01:37.259179Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:39.468227Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:40.037867Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/ -2021-02-21T12:01:40.617622Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/images -2021-02-21T12:01:41.211152Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/ -2021-02-21T12:01:41.827074Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/dockerfile/ -2021-02-21T12:01:42.416450Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:43.373285Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:43.962649Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/ -2021-02-21T12:01:44.533005Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/images -2021-02-21T12:01:45.101952Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/ -2021-02-21T12:01:45.786322Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/dockerfile/ -2021-02-21T12:01:46.371908Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:47.099819Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:47.665691Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/ -2021-02-21T12:01:48.299000Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/dockerfile/ -2021-02-21T12:01:48.885602Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:49.819503Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:50.397295Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/ -2021-02-21T12:01:50.974364Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/images -2021-02-21T12:01:51.543464Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/ -2021-02-21T12:01:52.170581Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/dockerfile/ -2021-02-21T12:01:52.754696Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:53.695898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:54.270786Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/ -2021-02-21T12:01:54.839574Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/images -2021-02-21T12:01:55.411184Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/ -2021-02-21T12:01:56.019008Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/dockerfile/ -2021-02-21T12:01:56.609886Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:01:57.507911Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:01:58.084891Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/ -2021-02-21T12:01:58.687249Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/images -2021-02-21T12:01:59.265319Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/ -2021-02-21T12:01:59.885904Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/dockerfile/ -2021-02-21T12:02:00.468241Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:01.354840Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:01.939761Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/ -2021-02-21T12:02:02.514483Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/images -2021-02-21T12:02:03.081585Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/ -2021-02-21T12:02:03.695852Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/dockerfile/ -2021-02-21T12:02:04.296640Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:05.269509Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:05.839006Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/ -2021-02-21T12:02:06.417601Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/images -2021-02-21T12:02:07.007467Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/ -2021-02-21T12:02:07.641890Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/dockerfile/ -2021-02-21T12:02:08.225307Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:09.134406Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:09.719310Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/ -2021-02-21T12:02:10.312800Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/images -2021-02-21T12:02:10.880603Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/ -2021-02-21T12:02:11.502283Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/dockerfile/ -2021-02-21T12:02:12.105061Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:13.093944Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:13.676530Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/ -2021-02-21T12:02:14.245021Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/images -2021-02-21T12:02:14.812879Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/ -2021-02-21T12:02:15.423901Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/dockerfile/ -2021-02-21T12:02:16.019170Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:16.965848Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:17.547897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/ -2021-02-21T12:02:18.117721Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/images -2021-02-21T12:02:18.692668Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/ -2021-02-21T12:02:19.335819Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/dockerfile/ -2021-02-21T12:02:19.934517Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:20.861891Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:21.430369Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/ -2021-02-21T12:02:21.999709Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/images -2021-02-21T12:02:22.573675Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/ -2021-02-21T12:02:23.184946Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/dockerfile/ -2021-02-21T12:02:23.767704Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:24.695186Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:25.266910Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/ -2021-02-21T12:02:25.848713Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/images -2021-02-21T12:02:26.433787Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/ -2021-02-21T12:02:27.067215Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/dockerfile/ -2021-02-21T12:02:27.659220Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:28.574469Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:29.147462Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/ -2021-02-21T12:02:29.721665Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/images -2021-02-21T12:02:30.294877Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/ -2021-02-21T12:02:30.915014Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/dockerfile/ -2021-02-21T12:02:31.497355Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:32.487894Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:33.066252Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/ -2021-02-21T12:02:33.633675Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/images -2021-02-21T12:02:34.206023Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/ -2021-02-21T12:02:34.822136Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/dockerfile/ -2021-02-21T12:02:35.439899Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-02-21T12:02:36.378949Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-02-21T12:02:36.952662Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/ -2021-02-21T12:02:37.538685Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/images -2021-02-21T12:02:37.573584Z Done with warcforceone +2021-02-21T14:00:01.197108Z Retrieving Docker Hub user archiveteam +2021-02-21T14:00:01.197433Z Fetching https://hub.docker.com/v2/users/archiveteam/ +2021-02-21T14:00:02.103150Z Fetching https://hub.docker.com/v2/repositories/archiveteam/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:03.866863Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/ +2021-02-21T14:00:04.509606Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/dockerfile/ +2021-02-21T14:00:05.475426Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:06.434170Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:07.025338Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/ +2021-02-21T14:00:07.596828Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/images +2021-02-21T14:00:07.647720Z Done with archiveteam +2021-02-21T14:00:07.648383Z Retrieving Docker Hub user warcforceone +2021-02-21T14:00:07.648532Z Fetching https://hub.docker.com/v2/users/warcforceone/ +2021-02-21T14:00:08.489198Z Fetching https://hub.docker.com/v2/repositories/warcforceone/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:09.951899Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/ +2021-02-21T14:00:10.558752Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/dockerfile/ +2021-02-21T14:00:11.142578Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:12.119284Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:12.695938Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/ +2021-02-21T14:00:13.279487Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/images +2021-02-21T14:00:13.873536Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/ +2021-02-21T14:00:14.495901Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/dockerfile/ +2021-02-21T14:00:15.085569Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:16.033507Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:16.623544Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/ +2021-02-21T14:00:17.190348Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/images +2021-02-21T14:00:17.751907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/ +2021-02-21T14:00:18.359896Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/dockerfile/ +2021-02-21T14:00:18.955391Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:19.847068Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:20.415826Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/ +2021-02-21T14:00:21.004830Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/images +2021-02-21T14:00:21.568577Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/ +2021-02-21T14:00:22.177288Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/dockerfile/ +2021-02-21T14:00:22.754244Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:23.699913Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:24.278883Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/ +2021-02-21T14:00:24.848754Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/images +2021-02-21T14:00:25.427679Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/ +2021-02-21T14:00:26.043898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/dockerfile/ +2021-02-21T14:00:26.624432Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:27.533700Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:28.119374Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/ +2021-02-21T14:00:28.691150Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/images +2021-02-21T14:00:29.273872Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/ +2021-02-21T14:00:29.887897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/dockerfile/ +2021-02-21T14:00:30.468574Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:31.393133Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:31.975348Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/ +2021-02-21T14:00:32.536775Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/images +2021-02-21T14:00:33.119952Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/ +2021-02-21T14:00:33.759903Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/dockerfile/ +2021-02-21T14:00:34.350661Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:35.287446Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:35.871147Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/ +2021-02-21T14:00:36.453756Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/images +2021-02-21T14:00:37.029550Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/ +2021-02-21T14:00:37.648600Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/dockerfile/ +2021-02-21T14:00:38.225005Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:39.163400Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:39.732906Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/ +2021-02-21T14:00:40.296525Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/images +2021-02-21T14:00:40.867468Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/ +2021-02-21T14:00:41.482466Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/dockerfile/ +2021-02-21T14:00:42.074872Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:42.996949Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:43.583697Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/ +2021-02-21T14:00:44.145592Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/images +2021-02-21T14:00:44.711118Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/ +2021-02-21T14:00:45.319165Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/dockerfile/ +2021-02-21T14:00:45.906830Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:46.829613Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:47.399192Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/ +2021-02-21T14:00:47.976098Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/images +2021-02-21T14:00:48.550082Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/ +2021-02-21T14:00:49.149949Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/dockerfile/ +2021-02-21T14:00:49.747447Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:50.685997Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:51.253146Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/ +2021-02-21T14:00:51.821501Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/images +2021-02-21T14:00:52.404572Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/ +2021-02-21T14:00:53.038126Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/dockerfile/ +2021-02-21T14:00:53.621398Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:54.587831Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:55.153070Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/ +2021-02-21T14:00:55.735557Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/images +2021-02-21T14:00:56.309785Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/ +2021-02-21T14:00:57.074202Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/dockerfile/ +2021-02-21T14:00:57.656070Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:00:58.591462Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:00:59.155921Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/ +2021-02-21T14:00:59.734702Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/images +2021-02-21T14:01:00.314078Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/ +2021-02-21T14:01:00.963913Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/dockerfile/ +2021-02-21T14:01:01.555342Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:02.681352Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:03.254250Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/ +2021-02-21T14:01:03.821196Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/images +2021-02-21T14:01:04.403480Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/ +2021-02-21T14:01:05.023827Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/dockerfile/ +2021-02-21T14:01:05.626212Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:06.565930Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:07.144224Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/ +2021-02-21T14:01:07.709768Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/images +2021-02-21T14:01:08.326277Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/ +2021-02-21T14:01:08.953506Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/dockerfile/ +2021-02-21T14:01:09.531418Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:10.226452Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:10.793049Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/ +2021-02-21T14:01:11.383313Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/images +2021-02-21T14:01:11.948949Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/ +2021-02-21T14:01:12.562078Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/dockerfile/ +2021-02-21T14:01:13.149728Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:14.083264Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:14.671292Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/ +2021-02-21T14:01:15.237692Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/images +2021-02-21T14:01:15.810780Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/ +2021-02-21T14:01:16.443897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/dockerfile/ +2021-02-21T14:01:17.047906Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:17.745584Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:18.331782Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/ +2021-02-21T14:01:18.949338Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/images +2021-02-21T14:01:19.515711Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/ +2021-02-21T14:01:20.153965Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/dockerfile/ +2021-02-21T14:01:20.751912Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:21.679547Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:22.247621Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/ +2021-02-21T14:01:22.817256Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/images +2021-02-21T14:01:23.386958Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/ +2021-02-21T14:01:24.029843Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/dockerfile/ +2021-02-21T14:01:24.612692Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:25.562915Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:26.127440Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/ +2021-02-21T14:01:26.704463Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/images +2021-02-21T14:01:27.283583Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/ +2021-02-21T14:01:27.915136Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/dockerfile/ +2021-02-21T14:01:28.510575Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:29.494819Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:30.065320Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/ +2021-02-21T14:01:30.635890Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/images +2021-02-21T14:01:31.201919Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/ +2021-02-21T14:01:31.829025Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/dockerfile/ +2021-02-21T14:01:32.422066Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:33.381598Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:33.971799Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/ +2021-02-21T14:01:34.542712Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/images +2021-02-21T14:01:35.121221Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/ +2021-02-21T14:01:35.748640Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/dockerfile/ +2021-02-21T14:01:36.322644Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:37.295323Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:37.885930Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/ +2021-02-21T14:01:38.454493Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/images +2021-02-21T14:01:39.051786Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/ +2021-02-21T14:01:39.683900Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/dockerfile/ +2021-02-21T14:01:40.278093Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:40.979521Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:41.542848Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/ +2021-02-21T14:01:42.150646Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/dockerfile/ +2021-02-21T14:01:42.739761Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:43.649727Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:44.231110Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/ +2021-02-21T14:01:44.794910Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/images +2021-02-21T14:01:45.375915Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/ +2021-02-21T14:01:45.995907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/dockerfile/ +2021-02-21T14:01:46.603917Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:47.524428Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:48.094534Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/ +2021-02-21T14:01:48.667901Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/images +2021-02-21T14:01:49.240675Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/ +2021-02-21T14:01:49.877771Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/dockerfile/ +2021-02-21T14:01:50.455305Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:51.451198Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:52.017056Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/ +2021-02-21T14:01:52.597550Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/images +2021-02-21T14:01:53.181564Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/ +2021-02-21T14:01:53.791305Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/dockerfile/ +2021-02-21T14:01:54.379899Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:55.349423Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:55.923906Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/ +2021-02-21T14:01:56.502803Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/images +2021-02-21T14:01:57.079159Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/ +2021-02-21T14:01:57.701367Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/dockerfile/ +2021-02-21T14:01:58.318705Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:01:59.223253Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:01:59.797161Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/ +2021-02-21T14:02:00.364528Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/images +2021-02-21T14:02:00.931251Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/ +2021-02-21T14:02:01.543298Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/dockerfile/ +2021-02-21T14:02:02.142315Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:03.112898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:03.682732Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/ +2021-02-21T14:02:04.255525Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/images +2021-02-21T14:02:04.821511Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/ +2021-02-21T14:02:05.434766Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/dockerfile/ +2021-02-21T14:02:06.018161Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:06.951901Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:07.536754Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/ +2021-02-21T14:02:08.104673Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/images +2021-02-21T14:02:08.669614Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/ +2021-02-21T14:02:09.291446Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/dockerfile/ +2021-02-21T14:02:09.865330Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:10.790542Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:11.359807Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/ +2021-02-21T14:02:11.930834Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/images +2021-02-21T14:02:12.503915Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/ +2021-02-21T14:02:13.126121Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/dockerfile/ +2021-02-21T14:02:13.709503Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:14.644369Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:15.219898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/ +2021-02-21T14:02:15.785036Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/images +2021-02-21T14:02:16.354094Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/ +2021-02-21T14:02:16.987002Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/dockerfile/ +2021-02-21T14:02:17.570006Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:18.622365Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:19.195348Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/ +2021-02-21T14:02:19.779901Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/images +2021-02-21T14:02:20.342796Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/ +2021-02-21T14:02:20.954591Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/dockerfile/ +2021-02-21T14:02:21.553838Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:22.501599Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:23.070237Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/ +2021-02-21T14:02:23.651153Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/images +2021-02-21T14:02:24.217093Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/ +2021-02-21T14:02:24.814994Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/dockerfile/ +2021-02-21T14:02:25.403289Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:26.343067Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:26.919750Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/ +2021-02-21T14:02:27.483911Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/images +2021-02-21T14:02:28.066355Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/ +2021-02-21T14:02:28.690859Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/dockerfile/ +2021-02-21T14:02:29.292590Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-02-21T14:02:30.249551Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-02-21T14:02:30.819182Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/ +2021-02-21T14:02:31.390785Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/images +2021-02-21T14:02:31.432935Z Done with warcforceone diff --git a/warcforceone/repositories.json b/warcforceone/repositories.json index 8c2c6d9edd..0f13e54584 100644 --- a/warcforceone/repositories.json +++ b/warcforceone/repositories.json @@ -10,7 +10,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 21053911, + "pull_count": 21054391, "last_updated": "2021-02-20T00:18:15.003220Z", "is_migrated": false, "collaborator_count": 0, @@ -82,7 +82,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 13725572, + "pull_count": 13727961, "last_updated": "2021-02-03T13:48:21.932838Z", "is_migrated": false, "collaborator_count": 0, diff --git a/warcforceone/repositories/dnsmasq/info.json b/warcforceone/repositories/dnsmasq/info.json index 6b66ce88d5..41a70308b8 100644 --- a/warcforceone/repositories/dnsmasq/info.json +++ b/warcforceone/repositories/dnsmasq/info.json @@ -9,7 +9,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 21053911, + "pull_count": 21054391, "last_updated": "2021-02-20T00:18:15.003220Z", "is_migrated": false, "collaborator_count": 0, diff --git a/warcforceone/repositories/dnsmasq/tags.json b/warcforceone/repositories/dnsmasq/tags.json index c70efc4ec5..004587b9cb 100644 --- a/warcforceone/repositories/dnsmasq/tags.json +++ b/warcforceone/repositories/dnsmasq/tags.json @@ -14,7 +14,7 @@ "os_version": null, "size": 4477304, "status": "active", - "last_pulled": "2021-02-21T11:24:17.380598Z", + "last_pulled": "2021-02-21T13:24:36.961025Z", "last_pushed": "2021-02-20T00:18:14.704711Z" } ], @@ -26,7 +26,7 @@ "full_size": 4477304, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-02-21T11:24:17.380598Z", + "tag_last_pulled": "2021-02-21T13:24:36.961025Z", "tag_last_pushed": "2021-02-20T00:18:14.704711Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/dnsmasq/tags/latest.images.json b/warcforceone/repositories/dnsmasq/tags/latest.images.json index 724154f200..97ee8e83c2 100644 --- a/warcforceone/repositories/dnsmasq/tags/latest.images.json +++ b/warcforceone/repositories/dnsmasq/tags/latest.images.json @@ -38,7 +38,7 @@ "os_version": null, "size": 4477304, "status": "active", - "last_pulled": "2021-02-21T11:24:17.380598Z", + "last_pulled": "2021-02-21T13:24:36.961025Z", "last_pushed": "2021-02-20T00:18:14.704711Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/dnsmasq/tags/latest.info.json b/warcforceone/repositories/dnsmasq/tags/latest.info.json index d706ccdb39..63dc9e1aeb 100644 --- a/warcforceone/repositories/dnsmasq/tags/latest.info.json +++ b/warcforceone/repositories/dnsmasq/tags/latest.info.json @@ -13,7 +13,7 @@ "os_version": null, "size": 4477304, "status": "active", - "last_pulled": "2021-02-21T11:24:17.380598Z", + "last_pulled": "2021-02-21T13:24:36.961025Z", "last_pushed": "2021-02-20T00:18:14.704711Z" } ], @@ -25,6 +25,6 @@ "full_size": 4477304, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-02-21T11:24:17.380598Z", + "tag_last_pulled": "2021-02-21T13:24:36.961025Z", "tag_last_pushed": "2021-02-20T00:18:14.704711Z" } \ No newline at end of file diff --git a/warcforceone/repositories/github-grab/info.json b/warcforceone/repositories/github-grab/info.json index 9357520383..92bca10b29 100644 --- a/warcforceone/repositories/github-grab/info.json +++ b/warcforceone/repositories/github-grab/info.json @@ -9,7 +9,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 13725580, + "pull_count": 13727968, "last_updated": "2021-02-03T13:48:21.932838Z", "is_migrated": false, "collaborator_count": 0, diff --git a/warcforceone/repositories/github-grab/tags.json b/warcforceone/repositories/github-grab/tags.json index 1f3960a78d..32b259b06b 100644 --- a/warcforceone/repositories/github-grab/tags.json +++ b/warcforceone/repositories/github-grab/tags.json @@ -14,7 +14,7 @@ "os_version": null, "size": 355285549, "status": "active", - "last_pulled": "2021-02-21T11:12:10.356967Z", + "last_pulled": "2021-02-21T13:12:12.279871Z", "last_pushed": "2021-02-03T13:48:21.641084Z" } ], @@ -26,7 +26,7 @@ "full_size": 355285549, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-02-21T11:12:10.356967Z", + "tag_last_pulled": "2021-02-21T13:12:12.279871Z", "tag_last_pushed": "2021-02-03T13:48:21.641084Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/github-grab/tags/latest.images.json b/warcforceone/repositories/github-grab/tags/latest.images.json index d61e576747..e05ab7f454 100644 --- a/warcforceone/repositories/github-grab/tags/latest.images.json +++ b/warcforceone/repositories/github-grab/tags/latest.images.json @@ -148,7 +148,7 @@ "os_version": null, "size": 355285549, "status": "active", - "last_pulled": "2021-02-21T11:12:10.356967Z", + "last_pulled": "2021-02-21T13:12:12.279871Z", "last_pushed": "2021-02-03T13:48:21.641084Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/github-grab/tags/latest.info.json b/warcforceone/repositories/github-grab/tags/latest.info.json index f17e06b50b..ec0a8cd1d7 100644 --- a/warcforceone/repositories/github-grab/tags/latest.info.json +++ b/warcforceone/repositories/github-grab/tags/latest.info.json @@ -13,7 +13,7 @@ "os_version": null, "size": 355285549, "status": "active", - "last_pulled": "2021-02-21T11:12:10.356967Z", + "last_pulled": "2021-02-21T13:12:12.279871Z", "last_pushed": "2021-02-03T13:48:21.641084Z" } ], @@ -25,6 +25,6 @@ "full_size": 355285549, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-02-21T11:12:10.356967Z", + "tag_last_pulled": "2021-02-21T13:12:12.279871Z", "tag_last_pushed": "2021-02-03T13:48:21.641084Z" } \ No newline at end of file