From 18d83bfcbe2d1d001c361ca38f189f27be67e86a Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist's bot Date: Wed, 3 Mar 2021 03:02:34 +0000 Subject: [PATCH] Automatic run --- .tmp.7qbM4g7gLs/archiveteam/repositories.json | 20 + .../warrior-dockerfile/Dockerfile | 80 +++ .../repositories/warrior-dockerfile/info.json | 25 + .../warrior-dockerfile/sources.json | 18 + .../repositories/warrior-dockerfile/tags.json | 32 + .../tags/latest.images.json | 170 +++++ .../warrior-dockerfile/tags/latest.info.json | 30 + .tmp.7qbM4g7gLs/archiveteam/user.json | 12 + .../dockerhub-metadata.retrieve.log | 239 +++++++ .../warcforceone/repositories.json | 668 ++++++++++++++++++ .../repositories/angelfire-grab/Dockerfile | 3 + .../repositories/angelfire-grab/info.json | 25 + .../repositories/angelfire-grab/sources.json | 18 + .../repositories/angelfire-grab/tags.json | 32 + .../angelfire-grab/tags/latest.images.json | 115 +++ .../angelfire-grab/tags/latest.info.json | 30 + .../repositories/ateam-airsync-meta/info.json | 25 + .../ateam-airsync-meta/sources.json | 1 + .../repositories/ateam-airsync-meta/tags.json | 32 + .../tags/latest.images.json | 62 ++ .../ateam-airsync-meta/tags/latest.info.json | 30 + .../repositories/ateam-airsync/info.json | 25 + .../repositories/ateam-airsync/sources.json | 1 + .../repositories/ateam-airsync/tags.json | 32 + .../ateam-airsync/tags/latest.images.json | 40 ++ .../ateam-airsync/tags/latest.info.json | 30 + .../repositories/bitbucket-grab/Dockerfile | 9 + .../repositories/bitbucket-grab/info.json | 25 + .../repositories/bitbucket-grab/sources.json | 18 + .../repositories/bitbucket-grab/tags.json | 32 + .../bitbucket-grab/tags/latest.images.json | 143 ++++ .../bitbucket-grab/tags/latest.info.json | 30 + .../repositories/clutch-grab/Dockerfile | 3 + .../repositories/clutch-grab/info.json | 25 + .../repositories/clutch-grab/sources.json | 18 + .../repositories/clutch-grab/tags.json | 32 + .../clutch-grab/tags/latest.images.json | 130 ++++ .../clutch-grab/tags/latest.info.json | 30 + .../repositories/clutchvideos-grab/Dockerfile | 3 + .../repositories/clutchvideos-grab/info.json | 25 + .../clutchvideos-grab/sources.json | 18 + .../repositories/clutchvideos-grab/tags.json | 32 + .../clutchvideos-grab/tags/latest.images.json | 130 ++++ .../clutchvideos-grab/tags/latest.info.json | 30 + .../repositories/dnsmasq/Dockerfile | 6 + .../repositories/dnsmasq/info.json | 25 + .../repositories/dnsmasq/sources.json | 18 + .../repositories/dnsmasq/tags.json | 32 + .../dnsmasq/tags/latest.images.json | 44 ++ .../dnsmasq/tags/latest.info.json | 30 + .../repositories/domains-grab/Dockerfile | 4 + .../repositories/domains-grab/info.json | 25 + .../repositories/domains-grab/sources.json | 18 + .../repositories/domains-grab/tags.json | 32 + .../domains-grab/tags/latest.images.json | 159 +++++ .../domains-grab/tags/latest.info.json | 30 + .../repositories/fotoalbum-grab/Dockerfile | 3 + .../repositories/fotoalbum-grab/info.json | 25 + .../repositories/fotoalbum-grab/sources.json | 18 + .../repositories/fotoalbum-grab/tags.json | 32 + .../fotoalbum-grab/tags/latest.images.json | 154 ++++ .../fotoalbum-grab/tags/latest.info.json | 30 + .../repositories/github-grab/Dockerfile | 3 + .../repositories/github-grab/info.json | 25 + .../repositories/github-grab/sources.json | 18 + .../repositories/github-grab/tags.json | 32 + .../github-grab/tags/latest.images.json | 154 ++++ .../github-grab/tags/latest.info.json | 30 + .../repositories/google-sites-grab/Dockerfile | 3 + .../repositories/google-sites-grab/info.json | 25 + .../google-sites-grab/sources.json | 18 + .../repositories/google-sites-grab/tags.json | 32 + .../google-sites-grab/tags/latest.images.json | 154 ++++ .../google-sites-grab/tags/latest.info.json | 30 + .../repositories/grab-base/Dockerfile | 15 + .../repositories/grab-base/info.json | 25 + .../repositories/grab-base/sources.json | 18 + .../repositories/grab-base/tags.json | 32 + .../grab-base/tags/latest.images.json | 105 +++ .../grab-base/tags/latest.info.json | 30 + .../repositories/megawarc-factory/Dockerfile | 12 + .../repositories/megawarc-factory/info.json | 25 + .../megawarc-factory/sources.json | 18 + .../repositories/megawarc-factory/tags.json | 32 + .../megawarc-factory/tags/latest.images.json | 64 ++ .../megawarc-factory/tags/latest.info.json | 30 + .../repositories/mercurial-grab/Dockerfile | 9 + .../repositories/mercurial-grab/info.json | 25 + .../repositories/mercurial-grab/sources.json | 18 + .../repositories/mercurial-grab/tags.json | 32 + .../mercurial-grab/tags/latest.images.json | 164 +++++ .../mercurial-grab/tags/latest.info.json | 30 + .../microsoft-download-center-grab/Dockerfile | 9 + .../microsoft-download-center-grab/info.json | 25 + .../sources.json | 18 + .../microsoft-download-center-grab/tags.json | 32 + .../tags/latest.images.json | 143 ++++ .../tags/latest.info.json | 30 + .../repositories/mixer-grab/Dockerfile | 9 + .../repositories/mixer-grab/info.json | 25 + .../repositories/mixer-grab/sources.json | 18 + .../repositories/mixer-grab/tags.json | 32 + .../mixer-grab/tags/latest.images.json | 143 ++++ .../mixer-grab/tags/latest.info.json | 30 + .../repositories/naver-matome-grab/Dockerfile | 3 + .../repositories/naver-matome-grab/info.json | 25 + .../naver-matome-grab/sources.json | 18 + .../repositories/naver-matome-grab/tags.json | 32 + .../naver-matome-grab/tags/latest.images.json | 138 ++++ .../naver-matome-grab/tags/latest.info.json | 30 + .../repositories/pastebin-grab/Dockerfile | 3 + .../repositories/pastebin-grab/info.json | 25 + .../repositories/pastebin-grab/sources.json | 18 + .../repositories/pastebin-grab/tags.json | 32 + .../pastebin-grab/tags/latest.images.json | 154 ++++ .../pastebin-grab/tags/latest.info.json | 30 + .../repositories/playstv-grab/Dockerfile | 5 + .../repositories/playstv-grab/info.json | 25 + .../repositories/playstv-grab/sources.json | 18 + .../repositories/reddit-grab/Dockerfile | 8 + .../repositories/reddit-grab/info.json | 25 + .../repositories/reddit-grab/sources.json | 16 + .../repositories/reddit-grab/tags.json | 32 + .../reddit-grab/tags/latest.images.json | 159 +++++ .../reddit-grab/tags/latest.info.json | 30 + .../repositories/singstar-grab/Dockerfile | 5 + .../repositories/singstar-grab/info.json | 25 + .../repositories/singstar-grab/sources.json | 18 + .../repositories/singstar-grab/tags.json | 32 + .../singstar-grab/tags/latest.images.json | 138 ++++ .../singstar-grab/tags/latest.info.json | 30 + .../repositories/snscrape-upstream/info.json | 25 + .../snscrape-upstream/sources.json | 1 + .../repositories/snscrape-upstream/tags.json | 1 + .../repositories/snscrape/Dockerfile | 9 + .../repositories/snscrape/info.json | 25 + .../repositories/snscrape/sources.json | 18 + .../repositories/snscrape/tags.json | 32 + .../snscrape/tags/latest.images.json | 101 +++ .../snscrape/tags/latest.info.json | 30 + .../repositories/soup-io-grab/Dockerfile | 9 + .../repositories/soup-io-grab/info.json | 25 + .../repositories/soup-io-grab/sources.json | 18 + .../repositories/soup-io-grab/tags.json | 32 + .../soup-io-grab/tags/latest.images.json | 143 ++++ .../soup-io-grab/tags/latest.info.json | 30 + .../tencent-weibo-grab/Dockerfile | 4 + .../repositories/tencent-weibo-grab/info.json | 25 + .../tencent-weibo-grab/sources.json | 18 + .../repositories/tencent-weibo-grab/tags.json | 32 + .../tags/latest.images.json | 143 ++++ .../tencent-weibo-grab/tags/latest.info.json | 30 + .../theartistunion-grab/Dockerfile | 9 + .../theartistunion-grab/info.json | 25 + .../theartistunion-grab/sources.json | 18 + .../theartistunion-grab/tags.json | 32 + .../tags/latest.images.json | 143 ++++ .../theartistunion-grab/tags/latest.info.json | 30 + .../repositories/yahoogroups-grab/Dockerfile | 5 + .../repositories/yahoogroups-grab/info.json | 25 + .../yahoogroups-grab/sources.json | 18 + .../repositories/yahoogroups-grab/tags.json | 32 + .../yahoogroups-grab/tags/latest.images.json | 138 ++++ .../yahoogroups-grab/tags/latest.info.json | 30 + .../youtube-playlistnotes-grab/Dockerfile | 3 + .../youtube-playlistnotes-grab/info.json | 25 + .../youtube-playlistnotes-grab/sources.json | 18 + .../youtube-playlistnotes-grab/tags.json | 32 + .../tags/latest.images.json | 154 ++++ .../tags/latest.info.json | 30 + .tmp.7qbM4g7gLs/warcforceone/user.json | 12 + archiveteam/repositories.json | 2 +- .../repositories/warrior-dockerfile/info.json | 2 +- .../repositories/warrior-dockerfile/tags.json | 4 +- .../tags/latest.images.json | 2 +- .../warrior-dockerfile/tags/latest.info.json | 4 +- dockerhub-metadata.retrieve.log | 468 ++++++------ warcforceone/repositories.json | 4 +- warcforceone/repositories/dnsmasq/info.json | 2 +- warcforceone/repositories/dnsmasq/tags.json | 4 +- .../dnsmasq/tags/latest.images.json | 2 +- .../dnsmasq/tags/latest.info.json | 4 +- .../repositories/github-grab/info.json | 2 +- .../repositories/github-grab/tags.json | 4 +- .../github-grab/tags/latest.images.json | 2 +- .../github-grab/tags/latest.info.json | 4 +- 186 files changed, 7794 insertions(+), 255 deletions(-) create mode 100644 .tmp.7qbM4g7gLs/archiveteam/repositories.json create mode 100644 .tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/info.json create mode 100644 .tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/sources.json create mode 100644 .tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags.json create mode 100644 .tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/archiveteam/user.json create mode 100644 .tmp.7qbM4g7gLs/dockerhub-metadata.retrieve.log create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/Dockerfile create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/sources.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.images.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.info.json create mode 100644 .tmp.7qbM4g7gLs/warcforceone/user.json diff --git a/.tmp.7qbM4g7gLs/archiveteam/repositories.json b/.tmp.7qbM4g7gLs/archiveteam/repositories.json new file mode 100644 index 0000000000..d53c76fbc7 --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/repositories.json @@ -0,0 +1,20 @@ +[ + { + "user": "archiveteam", + "name": "warrior-dockerfile", + "namespace": "archiveteam", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": true, + "can_edit": false, + "star_count": 16, + "pull_count": 10956274, + "last_updated": "2018-12-17T00:42:59.374566Z", + "is_migrated": true, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "archiveteam" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/Dockerfile b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/Dockerfile new file mode 100644 index 0000000000..f81f4c30cd --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/Dockerfile @@ -0,0 +1,80 @@ +# Use phusion/baseimage as base image. +FROM phusion/baseimage:0.11 + +# Use baseimage-docker's init system. +CMD ["/sbin/my_init"] + +ADD get-wget-lua.sh / + +# Install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + autoconf \ + flex \ + gcc \ + git \ + isc-dhcp-client \ + jq \ + libgnutls28-dev \ + liblua5.1-0 \ + liblua5.1-0-dev \ + make \ + net-tools \ + pciutils \ + python \ + python-pip \ + python-setuptools \ + python3 \ + python3-pip \ + python3-setuptools \ + rsync \ + software-properties-common \ + sudo \ + wget \ + && chmod +x /get-wget-lua.sh && sync && bash -c "/get-wget-lua.sh" \ + && apt-get remove -y \ + autoconf \ + flex \ + gcc \ + libgnutls28-dev \ + liblua5.1-0-dev \ + make \ + && apt-get clean && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Setup system for the warrior +RUN useradd warrior +RUN mkdir /home/warrior && chown warrior: /home/warrior + +# Clone warrior code +USER warrior +WORKDIR /home/warrior +RUN git clone -b docker https://github.com/ArchiveTeam/warrior-code2.git +USER root + +# Add the boot script (this will install the actual warrior on boot) +RUN mkdir -p /etc/my_init.d +ADD boot.sh /etc/my_init.d/warrior-boot.sh + +# running as root and/or requiring sudo is a bad practice in docker containers, +# sadly sudo is hard-coded all over the place in `warrior-code2` +RUN echo "warrior ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +# Expose web interface port +EXPOSE 8001 + +# Add the warrior service entry for runit +RUN mkdir /etc/service/warrior +ADD warrior.sh /etc/service/warrior/run + +# ENV to JSON +ADD env-to-json.sh /home/warrior + +ENV DOWNLOADER="" +ENV HTTP_PASSWORD="" +ENV HTTP_USERNAME="" +ENV SELECTED_PROJECT="" +ENV SHARED_RSYNC_THREADS="" +ENV WARRIOR_ID="" +ENV CONCURRENT_ITEMS="" + diff --git a/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/info.json b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/info.json new file mode 100644 index 0000000000..bc91ce05b1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/info.json @@ -0,0 +1,25 @@ +{ + "user": "archiveteam", + "name": "warrior-dockerfile", + "namespace": "archiveteam", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": true, + "can_edit": false, + "star_count": 16, + "pull_count": 10956274, + "last_updated": "2018-12-17T00:42:59.374566Z", + "is_migrated": true, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "archiveteam", + "has_starred": false, + "full_description": "## A Dockerfile for the [ArchiveTeam Warrior](https://www.archiveteam.org/index.php?title=ArchiveTeam_Warrior)\n\"Warrior\"Docker\n\nBuild, run, grab the container IP and access the web interface on port 8001.\n\nAvailable as a Trusted Build on the index as [`archiveteam/warrior-dockerfile`](https://index.docker.io/u/archiveteam/warrior-dockerfile/) so you can just\n\n```\ndocker pull archiveteam/warrior-dockerfile\n# run without -d to follow the warrior install process\n# you will need to detach or stop-and-start the container.\n# use -p to bind port 8001 on the docker container\n# (default ip 172.17.0.x) to port 8001 on localhost.\ndocker run [-d] [-p 127.0.0.1:8001:8001] archiveteam/warrior-dockerfile\n```\n\nIf you prefer to just run the process in the background, and automatically start it again after machine reboot, use this instead:\n\n``` shell-interaction\ndocker run --detach \\\n --publish 127.0.0.1:8001:8001 \\\n --restart always \\\n archiveteam/warrior-dockerfile\n```\n\n\n### Configuration\n\n\n#### Manual Using the Web Interface\nTo access the web interface get the container IP from `docker inspect` and point your browser to `http://IP:8001`. If you are running this container on a headless machine, be sure to bind the docker container's port to a port on that machine (e.g. `-p 8001:8001`) so that you can access the web interface on your LAN.\n\nYou can stop and resume the Warrior with `docker stop` and `docker start`\n\n\n#### Using Environment Variables\n\nIf you don't mount a `projects.json` configuration, you can provide seed settings using\nenvironment variables. Once a `projects.json` file exists, environment variables\nwill be ignored. Please note: This is currently not available in the Raspberry PI image.\n\n##### Example:\n\n```shell\n docker run \\\n --detach \\\n --env DOWNLOADER=\"your name\" -e \\\n --env SELECTED_PROJECT=\"auto\" \\\n --publish 8001:8001 \\\n --restart always \\\n archiveteam/warrior-dockerfile\n```\n\n##### Mapping\n\n| ENV | JSON key | Example | Default |\n|----------------------|----------------------|-------------------|---------|\n| DOWNLOADER | downloader | | |\n| HTTP_PASSWORD | http_password | | |\n| HTTP_USERNAME | http_username | | |\n| SELECTED_PROJECT | selected_project | `auto`, `tumblr` | |\n| SHARED_RSYNC_THREADS | shared:rsync_threads | | `20` |\n| WARRIOR_ID | warrior_id | | |\n| CONCURRENT_ITEMS | concurrent_items | | `3` |\n\n## Raspberry Pi\nYou can build the container with the following command:\n``` shell-interaction\ndocker build --rm -t warrior-arm32v5:latest -f Dockerfile.raspberry .\n```\n\nThe image needs a place to store the downloaded data as well as its\nconfiguration. Say you have a location suitable at /var/local/warrior\nuse the command below, otherwise update the data and config.json paths.\n\nFirst, create an empty config.json if it doesn't exist. Otherwise when you\nmount the path with docker it will create it as a directory.\n``` shell-interaction\ntouch /var/local/warrior/config.json\n```\n\nNow start the container.\n``` shell-interaction\ndocker run \\\n\t--volume /var/local/warrior/data:/data/data \\\n\t--volume /var/local/warrior/config.json:/home/warrior/projects/config.json \\\n\t--publish 8001:8001 \\\n\t--restart always \\\n\twarrior-arm32v5:latest\n```\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/sources.json b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/sources.json new file mode 100644 index 0000000000..75fc8f4ff5 --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/d7c66b31-70d8-4d86-b08b-256bc3f48c47/" + ], + "channel": "Stable", + "image": "archiveteam/warrior-dockerfile", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "warrior-dockerfile", + "resource_uri": "/api/build/v1/source/db2598ef-71cc-4786-a83b-ac055d36ba67/", + "state": "Empty", + "uuid": "db2598ef-71cc-4786-a83b-ac055d36ba67" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags.json b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags.json new file mode 100644 index 0000000000..735d6d4aaf --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6731, + "id": 7530, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:742864ef616f0a80bdd162538964c43df677f0912716ebda44605063258a6d18", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 119940149, + "status": "active", + "last_pulled": "2021-03-03T01:03:55.075398Z", + "last_pushed": null + } + ], + "last_updated": "2018-12-17T00:42:59.092382Z", + "last_updater": 6731, + "last_updater_username": "archiveteam", + "name": "latest", + "repository": 4436, + "full_size": 119940149, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T01:03:55.075398Z", + "tag_last_pushed": "2018-12-17T00:42:59.092382Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json new file mode 100644 index 0000000000..e3f737a9f4 --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json @@ -0,0 +1,170 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:742864ef616f0a80bdd162538964c43df677f0912716ebda44605063258a6d18", + "layers": [ + { + "digest": "sha256:c64513b741452f95d8a147b69c30f403f6289542dd7b2b51dd8ba0cb35d0e08b", + "size": 31658876, + "instruction": "ADD file:4bb62bb05874068552efeb626f8b31b4a29f26d6cc8c7d7fc7ab4c1fdece957a in / " + }, + { + "digest": "sha256:01b8b12bad90b51d9f15dd4b63103ea6221b339ac3b3e75807c963e678f28624", + "size": 847, + "instruction": "/bin/sh -c set -xe \t\t&& echo '#!/bin/sh' > /usr/sbin/policy-rc.d \t&& echo 'exit 101' >> /usr/sbin/policy-rc.d \t&& chmod +x /usr/sbin/policy-rc.d \t\t&& dpkg-divert --local --rename --add /sbin/initctl \t&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \t&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \t\t&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \t\t&& echo 'DPkg::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' > /etc/apt/apt.conf.d/docker-clean \t&& echo 'APT::Update::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' >> /etc/apt/apt.conf.d/docker-clean \t&& echo 'Dir::Cache::pkgcache \"\"; Dir::Cache::srcpkgcache \"\";' >> /etc/apt/apt.conf.d/docker-clean \t\t&& echo 'Acquire::Languages \"none\";' > /etc/apt/apt.conf.d/docker-no-languages \t\t&& echo 'Acquire::GzipIndexes \"true\"; Acquire::CompressionTypes::Order:: \"gz\";' > /etc/apt/apt.conf.d/docker-gzip-indexes \t\t&& echo 'Apt::AutoRemove::SuggestsImportant \"false\";' > /etc/apt/apt.conf.d/docker-autoremove-suggests" + }, + { + "digest": "sha256:c5d85cf7a05fec99bb829db84dc5a21cc0aca569253f45d1ea10ca9e8a03fa9a", + "size": 468, + "instruction": "/bin/sh -c rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b6b268720157210d21bbe49f6112f815774e6d2a6144b14911749fadfdb034f0", + "size": 849, + "instruction": "/bin/sh -c sed -i 's/^#\\s*\\(deb.*universe\\)$/\\1/g' /etc/apt/sources.list" + }, + { + "digest": "sha256:e12192999ff18f01315563c63333d7c1059cd8e64dffe75fffe504b95eeb093c", + "size": 163, + "instruction": "/bin/sh -c mkdir -p /run/systemd && echo 'docker' > /run/systemd/container" + }, + { + "size": 0, + "instruction": " CMD [\"/bin/bash\"]" + }, + { + "size": 0, + "instruction": " MAINTAINER Phusion " + }, + { + "digest": "sha256:d39ece66b66711cd10d00ecca889e6d9471a6c746abd4b1864d5159396013b07", + "size": 15045, + "instruction": "COPY dir:4156d6d2c4b57f6457db9076a456c428e0bc9deee73f34aaba10f3045eedeb35 in /bd_build " + }, + { + "digest": "sha256:65599be66378ba1b7638eb7f0ff71dc10f857d45ad289e61627e1256b3c52945", + "size": 35933387, + "instruction": "/bin/sh -c /bd_build/prepare.sh && \t/bd_build/system_services.sh && \t/bd_build/utilities.sh && \t/bd_build/cleanup.sh" + }, + { + "size": 0, + "instruction": " ENV DEBIAN_FRONTEND=teletype LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8" + }, + { + "size": 0, + "instruction": " CMD [\"/sbin/my_init\"]" + }, + { + "size": 0, + "instruction": " CMD [\"/sbin/my_init\"]" + }, + { + "digest": "sha256:5ce76b79e2fd9cff49fd987ba4b49641b5899ba24fa91893a5fdf7450c5d77ed", + "size": 673, + "instruction": "ADD file:5d94870fb53583fde1ef15d0c418754c94d42d0a82bc7b53e3dc93386a6cc064 in / " + }, + { + "digest": "sha256:9e96699e9e5ea169fd670896e04b4da6d193c143e030b5440f0e0c57442234f7", + "size": 52162287, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends autoconf flex gcc git isc-dhcp-client jq libgnutls28-dev liblua5.1-0 liblua5.1-0-dev make net-tools pciutils python python-pip python-setuptools python3 python3-pip python3-setuptools rsync software-properties-common sudo wget && chmod +x /get-wget-lua.sh && sync && bash -c \"/get-wget-lua.sh\" && apt-get remove -y autoconf flex gcc libgnutls28-dev liblua5.1-0-dev make && apt-get clean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*" + }, + { + "digest": "sha256:29870fd4e7bd112857984e8febf2d4c159b2088dcdac2df482073f76da9c3663", + "size": 2017, + "instruction": "/bin/sh -c useradd warrior" + }, + { + "digest": "sha256:29239dfdf917ce262ce161c99dfc4771868480db2fa98e64b61ad3513c898c78", + "size": 125, + "instruction": "/bin/sh -c mkdir /home/warrior && chown warrior: /home/warrior" + }, + { + "size": 0, + "instruction": " USER warrior" + }, + { + "size": 0, + "instruction": "WORKDIR /home/warrior" + }, + { + "digest": "sha256:99bc5d955aa52a839127a03297349b0205954c88b13cbc572c3d754f294e7b68", + "size": 163573, + "instruction": "/bin/sh -c git clone -b docker https://github.com/ArchiveTeam/warrior-code2.git" + }, + { + "size": 0, + "instruction": " USER root" + }, + { + "size": 0, + "instruction": "/bin/sh -c mkdir -p /etc/my_init.d" + }, + { + "digest": "sha256:b15ca0c6a79bf7c837e68bdb569e60f1d9c50372dd0c79a4bae6986b857cdd66", + "size": 259, + "instruction": "ADD file:47d258ef4e4ea1a817763d1496002fc6f8ddf531a760e99b078816f3196fccd6 in /etc/my_init.d/warrior-boot.sh " + }, + { + "digest": "sha256:6b59ed47645d2520c1677fdfe1ad21b75bc782518bd09f2125a3fa784a1bc353", + "size": 539, + "instruction": "/bin/sh -c echo \"warrior ALL=(ALL) NOPASSWD: ALL\" >> /etc/sudoers" + }, + { + "size": 0, + "instruction": " EXPOSE 8001" + }, + { + "digest": "sha256:a0be2c0f04b4ee626df02c7dd6d4379a10e6504a1f99f1737e9a9a96ef26f991", + "size": 184, + "instruction": "/bin/sh -c mkdir /etc/service/warrior" + }, + { + "digest": "sha256:c93d3f659e1baeb1bce005a4e692b95db8db3574f415a1c01133d1779136650e", + "size": 349, + "instruction": "ADD file:6a76fa03c473181539860b5a1806568aa61928c6fa5f0950e88306616d576d3c in /etc/service/warrior/run " + }, + { + "digest": "sha256:d80eda2ef6b14c973c0372a8f706de70324fb7a531196e7a3bd2e0c978b64f4e", + "size": 508, + "instruction": "ADD file:48a7d65247d6ff3054a26127cbe7816687abbc79b682b0442b660e7480187a6e in /home/warrior " + }, + { + "size": 0, + "instruction": " ENV DOWNLOADER=" + }, + { + "size": 0, + "instruction": " ENV HTTP_PASSWORD=" + }, + { + "size": 0, + "instruction": " ENV HTTP_USERNAME=" + }, + { + "size": 0, + "instruction": " ENV SELECTED_PROJECT=" + }, + { + "size": 0, + "instruction": " ENV SHARED_RSYNC_THREADS=" + }, + { + "size": 0, + "instruction": " ENV WARRIOR_ID=" + }, + { + "size": 0, + "instruction": " ENV CONCURRENT_ITEMS=" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 119940149, + "status": "active", + "last_pulled": "2021-03-03T01:03:55.075398Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json new file mode 100644 index 0000000000..f035229373 --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6731, + "id": 7530, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:742864ef616f0a80bdd162538964c43df677f0912716ebda44605063258a6d18", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 119940149, + "status": "active", + "last_pulled": "2021-03-03T01:03:55.075398Z", + "last_pushed": null + } + ], + "last_updated": "2018-12-17T00:42:59.092382Z", + "last_updater": 6731, + "last_updater_username": "archiveteam", + "name": "latest", + "repository": 4436, + "full_size": 119940149, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T01:03:55.075398Z", + "tag_last_pushed": "2018-12-17T00:42:59.092382Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/archiveteam/user.json b/.tmp.7qbM4g7gLs/archiveteam/user.json new file mode 100644 index 0000000000..7c66732e78 --- /dev/null +++ b/.tmp.7qbM4g7gLs/archiveteam/user.json @@ -0,0 +1,12 @@ +{ + "id": "2d4b1e3074c011e4bea40242ac11001b", + "orgname": "archiveteam", + "full_name": "", + "location": "", + "company": "", + "profile_url": "", + "date_joined": "2013-11-29T04:49:43Z", + "gravatar_url": "https://secure.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e.jpg?s=80&r=g&d=mm", + "gravatar_email": "", + "type": "Organization" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/dockerhub-metadata.retrieve.log b/.tmp.7qbM4g7gLs/dockerhub-metadata.retrieve.log new file mode 100644 index 0000000000..b119418838 --- /dev/null +++ b/.tmp.7qbM4g7gLs/dockerhub-metadata.retrieve.log @@ -0,0 +1,239 @@ +2021-03-03T02:00:01.562273Z Retrieving Docker Hub user archiveteam +2021-03-03T02:00:01.562554Z Fetching https://hub.docker.com/v2/users/archiveteam/ +2021-03-03T02:00:02.362244Z Fetching https://hub.docker.com/v2/repositories/archiveteam/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:03.015125Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/ +2021-03-03T02:00:04.276851Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/dockerfile/ +2021-03-03T02:00:05.251019Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:06.138923Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:06.722850Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/ +2021-03-03T02:00:07.290120Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/images +2021-03-03T02:00:07.327152Z Done with archiveteam +2021-03-03T02:00:07.327757Z Retrieving Docker Hub user warcforceone +2021-03-03T02:00:07.327920Z Fetching https://hub.docker.com/v2/users/warcforceone/ +2021-03-03T02:00:08.143910Z Fetching https://hub.docker.com/v2/repositories/warcforceone/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:08.853290Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/ +2021-03-03T02:00:09.487004Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/dockerfile/ +2021-03-03T02:00:10.078122Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:11.243239Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:11.826432Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/ +2021-03-03T02:00:12.400677Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/images +2021-03-03T02:00:12.984167Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/ +2021-03-03T02:00:13.512334Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/dockerfile/ +2021-03-03T02:00:14.123686Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:15.255429Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:15.840505Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/ +2021-03-03T02:00:16.406479Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/images +2021-03-03T02:00:16.978052Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/ +2021-03-03T02:00:17.606704Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/dockerfile/ +2021-03-03T02:00:18.183482Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:19.894808Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:20.477815Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/ +2021-03-03T02:00:21.050067Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/images +2021-03-03T02:00:21.625752Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/ +2021-03-03T02:00:22.248939Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/dockerfile/ +2021-03-03T02:00:22.843931Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:23.732678Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:24.297610Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/ +2021-03-03T02:00:24.880521Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/images +2021-03-03T02:00:25.465781Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/ +2021-03-03T02:00:26.093845Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/dockerfile/ +2021-03-03T02:00:26.675903Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:27.591907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:28.217433Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/ +2021-03-03T02:00:28.814233Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/images +2021-03-03T02:00:29.420847Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/ +2021-03-03T02:00:30.054834Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/dockerfile/ +2021-03-03T02:00:30.709047Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:31.876884Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:32.445939Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/ +2021-03-03T02:00:33.015399Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/images +2021-03-03T02:00:33.591284Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/ +2021-03-03T02:00:34.210557Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/dockerfile/ +2021-03-03T02:00:34.797940Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:35.753406Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:36.323453Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/ +2021-03-03T02:00:36.893093Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/images +2021-03-03T02:00:37.471902Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/ +2021-03-03T02:00:38.103904Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/dockerfile/ +2021-03-03T02:00:38.726887Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:39.703906Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:40.283047Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/ +2021-03-03T02:00:40.869544Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/images +2021-03-03T02:00:41.444768Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/ +2021-03-03T02:00:42.066240Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/dockerfile/ +2021-03-03T02:00:42.643907Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:43.644916Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:44.225107Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/ +2021-03-03T02:00:44.799083Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/images +2021-03-03T02:00:45.367913Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/ +2021-03-03T02:00:45.987152Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/dockerfile/ +2021-03-03T02:00:46.605211Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:47.550201Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:48.119901Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/ +2021-03-03T02:00:48.699912Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/images +2021-03-03T02:00:49.273413Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/ +2021-03-03T02:00:50.004999Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/dockerfile/ +2021-03-03T02:00:50.583899Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:51.470083Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:52.055217Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/ +2021-03-03T02:00:52.632582Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/images +2021-03-03T02:00:53.226993Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/ +2021-03-03T02:00:53.854177Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/dockerfile/ +2021-03-03T02:00:54.439150Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:55.286628Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:55.860830Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/ +2021-03-03T02:00:56.451849Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/images +2021-03-03T02:00:57.026153Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/ +2021-03-03T02:00:57.654490Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/dockerfile/ +2021-03-03T02:00:58.252841Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:00:59.360824Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:00:59.939546Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/ +2021-03-03T02:01:00.517254Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/images +2021-03-03T02:01:01.094198Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/ +2021-03-03T02:01:01.717808Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/dockerfile/ +2021-03-03T02:01:02.381074Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:03.542588Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:04.126115Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/ +2021-03-03T02:01:04.709393Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/images +2021-03-03T02:01:05.279917Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/ +2021-03-03T02:01:05.906044Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/dockerfile/ +2021-03-03T02:01:06.488953Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:07.413521Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:07.999908Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/ +2021-03-03T02:01:08.584737Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/images +2021-03-03T02:01:09.182506Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/ +2021-03-03T02:01:09.837567Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/dockerfile/ +2021-03-03T02:01:10.443894Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:11.136574Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:11.738123Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/ +2021-03-03T02:01:12.319501Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/images +2021-03-03T02:01:12.907862Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/ +2021-03-03T02:01:13.567897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/dockerfile/ +2021-03-03T02:01:14.168568Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:15.201618Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:15.779565Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/ +2021-03-03T02:01:16.394557Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/images +2021-03-03T02:01:16.976871Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/ +2021-03-03T02:01:17.609316Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/dockerfile/ +2021-03-03T02:01:18.195432Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:18.839233Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:19.450767Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/ +2021-03-03T02:01:20.049619Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/images +2021-03-03T02:01:20.634091Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/ +2021-03-03T02:01:21.271896Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/dockerfile/ +2021-03-03T02:01:21.863899Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:22.742754Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:23.318281Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/ +2021-03-03T02:01:23.936672Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/images +2021-03-03T02:01:24.511436Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/ +2021-03-03T02:01:25.119247Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/dockerfile/ +2021-03-03T02:01:25.715710Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:26.777756Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:27.409777Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/ +2021-03-03T02:01:28.023439Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/images +2021-03-03T02:01:28.666377Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/ +2021-03-03T02:01:29.341194Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/dockerfile/ +2021-03-03T02:01:30.066298Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:31.379535Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:31.951345Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/ +2021-03-03T02:01:32.547581Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/images +2021-03-03T02:01:33.155689Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/ +2021-03-03T02:01:33.779710Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/dockerfile/ +2021-03-03T02:01:34.387855Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:35.375587Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:35.963907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/ +2021-03-03T02:01:36.567141Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/images +2021-03-03T02:01:37.179064Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/ +2021-03-03T02:01:37.823263Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/dockerfile/ +2021-03-03T02:01:38.423896Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:39.337841Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:39.918595Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/ +2021-03-03T02:01:40.507895Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/images +2021-03-03T02:01:41.093152Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/ +2021-03-03T02:01:41.737727Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/dockerfile/ +2021-03-03T02:01:42.319271Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:42.940681Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:43.500777Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/ +2021-03-03T02:01:44.123910Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/dockerfile/ +2021-03-03T02:01:44.717437Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:45.660747Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:46.247710Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/ +2021-03-03T02:01:46.830629Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/images +2021-03-03T02:01:47.418835Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/ +2021-03-03T02:01:48.059230Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/dockerfile/ +2021-03-03T02:01:48.645691Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:49.622412Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:50.210385Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/ +2021-03-03T02:01:50.824486Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/images +2021-03-03T02:01:51.418693Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/ +2021-03-03T02:01:52.099019Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/dockerfile/ +2021-03-03T02:01:52.708935Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:01:57.906327Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T02:01:58.537241Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/ +2021-03-03T02:01:59.159907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/images +2021-03-03T02:01:59.787711Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/ +2021-03-03T02:02:00.446751Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/dockerfile/ +2021-03-03T02:02:01.058168Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T02:02:02.041533Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/?page_size=100&page=1&ordering=last_updated +Traceback (most recent call last): + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 706, in urlopen + chunked=chunked, + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 445, in _make_request + six.raise_from(e, None) + File "", line 3, in raise_from + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 440, in _make_request + httplib_response = conn.getresponse() + File "/usr/lib/python3.7/http/client.py", line 1336, in getresponse + response.begin() + File "/usr/lib/python3.7/http/client.py", line 306, in begin + version, status, reason = self._read_status() + File "/usr/lib/python3.7/http/client.py", line 275, in _read_status + raise RemoteDisconnected("Remote end closed connection without" +http.client.RemoteDisconnected: Remote end closed connection without response + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/atdockerhub/.local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send + timeout=timeout + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 756, in urlopen + method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/util/retry.py", line 531, in increment + raise six.reraise(type(error), error, _stacktrace) + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/packages/six.py", line 734, in reraise + raise value.with_traceback(tb) + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 706, in urlopen + chunked=chunked, + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 445, in _make_request + six.raise_from(e, None) + File "", line 3, in raise_from + File "/home/atdockerhub/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 440, in _make_request + httplib_response = conn.getresponse() + File "/usr/lib/python3.7/http/client.py", line 1336, in getresponse + response.begin() + File "/usr/lib/python3.7/http/client.py", line 306, in begin + version, status, reason = self._read_status() + File "/usr/lib/python3.7/http/client.py", line 275, in _read_status + raise RemoteDisconnected("Remote end closed connection without" +urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/atdockerhub/dockerhub-metadata-master/retrieve", line 151, in + main() + File "/home/atdockerhub/dockerhub-metadata-master/retrieve", line 144, in main + retrieve(profile) + File "/home/atdockerhub/dockerhub-metadata-master/retrieve", line 112, in retrieve + r = fetch(session, nextUrl) + File "/home/atdockerhub/dockerhub-metadata-master/retrieve", line 36, in fetch + r = session.get(url, **kwargs) + File "/home/atdockerhub/.local/lib/python3.7/site-packages/requests/sessions.py", line 555, in get + return self.request('GET', url, **kwargs) + File "/home/atdockerhub/.local/lib/python3.7/site-packages/requests/sessions.py", line 542, in request + resp = self.send(prep, **send_kwargs) + File "/home/atdockerhub/.local/lib/python3.7/site-packages/requests/sessions.py", line 655, in send + r = adapter.send(request, **kwargs) + File "/home/atdockerhub/.local/lib/python3.7/site-packages/requests/adapters.py", line 498, in send + raise ConnectionError(err, request=request) +requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories.json b/.tmp.7qbM4g7gLs/warcforceone/repositories.json new file mode 100644 index 0000000000..a0271be4d4 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories.json @@ -0,0 +1,668 @@ +[ + { + "user": "warcforceone", + "name": "dnsmasq", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 21109092, + "last_updated": "2021-02-20T00:18:15.003220Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "angelfire-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 412, + "last_updated": "2021-02-13T11:34:56.603218Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "grab-base", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 50530, + "last_updated": "2021-02-13T11:25:52.346081Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "google-sites-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7181229, + "last_updated": "2021-02-03T14:12:19.864923Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "github-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 13992740, + "last_updated": "2021-02-03T13:48:21.932838Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "pastebin-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 386445, + "last_updated": "2021-02-03T13:42:13.640186Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "reddit-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 5792737, + "last_updated": "2021-01-14T00:56:43.223341Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "domains-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 151168, + "last_updated": "2021-01-09T14:53:49.620078Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "megawarc-factory", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 80, + "last_updated": "2020-12-29T13:44:29.663201Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "fotoalbum-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 835123, + "last_updated": "2020-10-31T11:11:27.569458Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "youtube-playlistnotes-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11642978, + "last_updated": "2020-10-31T11:04:04.603216Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "mercurial-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 1139185, + "last_updated": "2020-10-31T10:57:10.413100Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "tencent-weibo-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 94271777, + "last_updated": "2020-10-31T00:33:22.019703Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "theartistunion-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 139872, + "last_updated": "2020-10-15T14:40:26.351961Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "bitbucket-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 976507, + "last_updated": "2020-10-15T14:32:38.301135Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "ateam-airsync", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 149, + "last_updated": "2020-09-29T20:52:21.216962Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "naver-matome-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 8478031, + "last_updated": "2020-09-22T15:42:01.244574Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "ateam-airsync-meta", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 197626, + "last_updated": "2020-09-04T01:36:11.426772Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "clutch-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11078089, + "last_updated": "2020-08-20T14:53:12.542994Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "clutchvideos-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 6455998, + "last_updated": "2020-08-15T01:12:03.411902Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "microsoft-download-center-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7473694, + "last_updated": "2020-08-03T20:33:39.400223Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "mixer-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 1299654, + "last_updated": "2020-07-23T17:26:35.210553Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "soup-io-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 935469, + "last_updated": "2020-07-21T16:00:50.757421Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "snscrape-upstream", + "namespace": "warcforceone", + "repository_type": null, + "status": 0, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 0, + "last_updated": null, + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "snscrape", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 21, + "last_updated": "2020-06-03T01:31:31.271076Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "singstar-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 23644, + "last_updated": "2020-01-21T23:35:49.829575Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "yahoogroups-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 251333, + "last_updated": "2019-12-12T17:52:18.779336Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "playstv-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 54289, + "last_updated": "2019-12-12T17:09:45.546166Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "youtube-likedlists-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 1, + "pull_count": 1489988, + "last_updated": "2019-12-09T23:22:16.580382Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "gfycat-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 491053, + "last_updated": "2019-11-28T23:21:38.456350Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "drawr-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 18578, + "last_updated": "2019-11-20T08:53:55.615761Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "radio24syv-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 56669, + "last_updated": "2019-11-02T21:19:03.786842Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "yourshot-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 28284, + "last_updated": "2019-10-30T01:49:43.816568Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "sketch-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 407761, + "last_updated": "2019-09-25T15:13:33.634433Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "tinypic-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7330350, + "last_updated": "2019-09-09T18:11:17.375929Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "freeml-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 22, + "last_updated": "2019-09-02T21:47:22.366328Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + }, + { + "user": "warcforceone", + "name": "instaudio-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 101, + "last_updated": "2019-08-03T02:38:42.937172Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/Dockerfile new file mode 100644 index 0000000000..c9b2c8d6ef --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-lua diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/info.json new file mode 100644 index 0000000000..10e5c88716 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "angelfire-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 412, + "last_updated": "2021-02-13T11:34:56.603218Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "angelfire-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [angelfire](http://archiveteam.org/index.php?title=angelfire)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"angelfire\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n pip install --upgrade seesaw\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/angelfire-grab.git; cd angelfire-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/angelfire-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y install autoconf automake flex gnutls-devel lua-devel python-pip zlib-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, angelfire-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/angelfire-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/sources.json new file mode 100644 index 0000000000..760acdfe5a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/5d60025b-29a9-461d-8285-0525769b886a/" + ], + "channel": "Stable", + "image": "warcforceone/angelfire-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "angelfire-grab", + "resource_uri": "/api/build/v1/source/0346b47f-beb6-4bb1-b437-dfac7e86521c/", + "state": "Success", + "uuid": "0346b47f-beb6-4bb1-b437-dfac7e86521c" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags.json new file mode 100644 index 0000000000..9ffb8b8071 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 63790251, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b512078401ce1016c06d80a6dcf38589d4b1d3f2faf42078bf0d39bbe8e0d32a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 86392705, + "status": "active", + "last_pulled": "2021-02-20T02:21:56.8533Z", + "last_pushed": "2021-02-13T11:34:56.278689Z" + } + ], + "last_updated": "2021-02-13T11:34:56.278689Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470931, + "full_size": 86392705, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:56.8533Z", + "tag_last_pushed": "2021-02-13T11:34:56.278689Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.images.json new file mode 100644 index 0000000000..a1b54f3a72 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.images.json @@ -0,0 +1,115 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:b512078401ce1016c06d80a6dcf38589d4b1d3f2faf42078bf0d39bbe8e0d32a", + "layers": [ + { + "digest": "sha256:45b42c59be334ecda0daaa139b2f7d310e45c564c5f12263b1b8e68ec9e810ed", + "size": 27095142, + "instruction": "ADD file:d5c41bfaf15180481d8606f50799297e3f49b8a258c7c2cd988ab2bf0013272d in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:8cd3485318dba8e325f6b665105e10c51f4011cf2e2b4378ae2b92e104ff3566", + "size": 2769103, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tnetbase \t\ttzdata \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:b430e1981978aa8083832db9dc3f927e4336fbccf20f27a0419a5d31f78205b7", + "size": 10888464, + "instruction": "/bin/sh -c set -ex \t\t&& savedAptMark=\"$(apt-mark showmanual)\" \t&& apt-get update && apt-get install -y --no-install-recommends \t\tdpkg-dev \t\tgcc \t\tlibbluetooth-dev \t\tlibbz2-dev \t\tlibc6-dev \t\tlibexpat1-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tliblzma-dev \t\tlibncursesw5-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tmake \t\ttk-dev \t\tuuid-dev \t\twget \t\txz-utils \t\tzlib1g-dev \t\t$(command -v gpg > /dev/null || echo 'gnupg dirmngr') \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tLDFLAGS=\"-Wl,--strip-all\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& apt-mark auto '.*' > /dev/null \t&& apt-mark manual $savedAptMark \t&& find /usr/local -type f -executable -not \\( -name '*tkinter*' \\) -exec ldd '{}' ';' \t\t| awk '/=>/ { print $(NF-1) }' \t\t| sort -u \t\t| xargs -r dpkg-query --search \t\t| cut -d: -f1 \t\t| sort -u \t\t| xargs -r apt-mark manual \t&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \t&& rm -rf /var/lib/apt/lists/* \t\t&& python3 --version" + }, + { + "digest": "sha256:37d877dc41c386154662483278f5b856d8c34e200d571eb41cee562506fa5bfb", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=21.0.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/4be3fe44ad9dedc028629ed1497052d65d281b8e/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=8006625804f55e1bd99ad4214fd07082fee27a1c35945648a58f9087a714e9d4" + }, + { + "digest": "sha256:0b70d4ba14ef92ccaa7839657e6fca3422f09041c50a44011d6a2382d0c23fe8", + "size": 2452248, + "instruction": "/bin/sh -c set -ex; \t\tsavedAptMark=\"$(apt-mark showmanual)\"; \tapt-get update; \tapt-get install -y --no-install-recommends wget; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tapt-mark auto '.*' > /dev/null; \t[ -z \"$savedAptMark\" ] || apt-mark manual $savedAptMark; \tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \trm -rf /var/lib/apt/lists/*; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "digest": "sha256:2a5987f3c885013cfbe6ee2d5fa8bf80cbc61323e0ac8aa63ab2720faa2b9531", + "size": 926625, + "instruction": "COPY file:c7a381cd8311209dda148a7d15ef252c496da848907afaa18f8371d09648021e in /usr/local/bin/wget-lua " + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:4c4e90548af1cdbeb6d8ce5cde1c907ac0a4e48aed698d9a80b2afd3d3128111", + "size": 41771879, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket libpsl5 git && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0c0bb03728b9b8735afce4e3653c7761db07f3245cfa66a44bda45f8022184c8", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:209ae7f910171062848d7b09e65cbfbae13be920960f918f7ffdc862ff777905", + "size": 488779, + "instruction": "COPY dir:4d7664ff6cefb66182fe887cf6427bbf680a6a2d23fa04b10c3acc01deba141c in /grab " + }, + { + "digest": "sha256:29ab74bab7311b2ab6d658aa350e38f2b151ff058f507f36c89668ee9f35ef80", + "size": 140, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-lua" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 86392705, + "status": "active", + "last_pulled": "2021-02-20T02:21:56.8533Z", + "last_pushed": "2021-02-13T11:34:56.278689Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.info.json new file mode 100644 index 0000000000..4b7321c402 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/angelfire-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 63790251, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b512078401ce1016c06d80a6dcf38589d4b1d3f2faf42078bf0d39bbe8e0d32a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 86392705, + "status": "active", + "last_pulled": "2021-02-20T02:21:56.8533Z", + "last_pushed": "2021-02-13T11:34:56.278689Z" + } + ], + "last_updated": "2021-02-13T11:34:56.278689Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470931, + "full_size": 86392705, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:56.8533Z", + "tag_last_pushed": "2021-02-13T11:34:56.278689Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/info.json new file mode 100644 index 0000000000..57f909b416 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "ateam-airsync-meta", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 197626, + "last_updated": "2020-09-04T01:36:11.426772Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/sources.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/sources.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags.json new file mode 100644 index 0000000000..9ee743dc4c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 1202997, + "id": 115894726, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:47fd629d3787d565cd20db1a66d76c08667a41c2c03daddabbc260eb4840cb48", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 30305756, + "status": "stale", + "last_pulled": "2021-01-24T11:47:34.569491Z", + "last_pushed": null + } + ], + "last_updated": "2020-09-04T01:36:11.0808Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 9759263, + "full_size": 30305756, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-24T11:47:34.569491Z", + "tag_last_pushed": "2020-09-04T01:36:11.0808Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.images.json new file mode 100644 index 0000000000..42254bbfff --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.images.json @@ -0,0 +1,62 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:47fd629d3787d565cd20db1a66d76c08667a41c2c03daddabbc260eb4840cb48", + "layers": [ + { + "digest": "sha256:e7c96db7181be991f19a9fb6975cdbbd73c65f4a2681348e63a141a2192a5f10", + "size": 2757034, + "instruction": "ADD file:a86aea1f3a7d68f6ae03397b99ea77f2e9ee901c5c59e59f76f93adbb4035913 in / " + }, + { + "size": 0, + "instruction": " CMD [\"/bin/sh\"]" + }, + { + "size": 0, + "instruction": " ENV NODE_VERSION=12.2.0" + }, + { + "digest": "sha256:a9b145f64bbe7efacd482b8576a2098dd7054c7d612a9ab0ab06570b4ae2bb0d", + "size": 22472612, + "instruction": "/bin/sh -c addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node && apk add --no-cache libstdc++ && apk add --no-cache --virtual .build-deps binutils-gold curl g++ gcc gnupg libgcc linux-headers make python && for key in 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 FD3A5288F042B6850C66B31F09FE44734EB7990E 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 DD8F2338BAE7501E3DD5AC78C273792F7D83545D C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 B9AE9905FFD7803F25714661B63B535A4C206CA9 77984A986EBC2AA786BC0F66B01FBB92821C587A 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 4ED778F539E3634C779C87C6D7062848A1AB005C A48C2BEE680E841632CD4E44F07496B3EB3C1762 B9E2F5981AA6E0CD28160D9FF13993A75599653C ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz\" && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc\" && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && grep \" node-v$NODE_VERSION.tar.xz\\$\" SHASUMS256.txt | sha256sum -c - && tar -xf \"node-v$NODE_VERSION.tar.xz\" && cd \"node-v$NODE_VERSION\" && ./configure && make -j$(getconf _NPROCESSORS_ONLN) V= && make install && apk del .build-deps && cd .. && rm -Rf \"node-v$NODE_VERSION\" && rm \"node-v$NODE_VERSION.tar.xz\" SHASUMS256.txt.asc SHASUMS256.txt" + }, + { + "size": 0, + "instruction": " ENV YARN_VERSION=1.15.2" + }, + { + "digest": "sha256:3bcb5e14be535bd6a0d2366436d93d11b56bf9340365d25eb98f1397aec23470", + "size": 1333853, + "instruction": "/bin/sh -c apk add --no-cache --virtual .build-deps-yarn curl gnupg tar && for key in 6A010C5166006599AA17F08146C2130DFD2497F5 ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz\" && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc\" && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && mkdir -p /opt && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && apk del .build-deps-yarn" + }, + { + "size": 0, + "instruction": " CMD [\"node\"]" + }, + { + "digest": "sha256:de4ba1e42d5621c17db371c4c92fd649aaf691be6a5af0b327d617a01191f5af", + "size": 3741240, + "instruction": "/bin/sh -c apk upgrade --no-cache && apk add --no-cache bash coreutils" + }, + { + "digest": "sha256:f093645e5eb6f0884bb1f5d981582ea0ebd64322095e11c71950659d001000fb", + "size": 1017, + "instruction": "COPY file:e28d1f2938be3ca1f4b959a73d71e4d4b49fe3c5d7975bcd58a06511906ddbcf in / " + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"/index.js\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 30305756, + "status": "stale", + "last_pulled": "2021-01-24T11:47:34.569491Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.info.json new file mode 100644 index 0000000000..a17e6ad505 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync-meta/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 1202997, + "id": 115894726, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:47fd629d3787d565cd20db1a66d76c08667a41c2c03daddabbc260eb4840cb48", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 30305756, + "status": "stale", + "last_pulled": "2021-01-24T11:47:34.569491Z", + "last_pushed": null + } + ], + "last_updated": "2020-09-04T01:36:11.0808Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 9759263, + "full_size": 30305756, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-24T11:47:34.569491Z", + "tag_last_pushed": "2020-09-04T01:36:11.0808Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/info.json new file mode 100644 index 0000000000..19e7bb048c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "ateam-airsync", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 149, + "last_updated": "2020-09-29T20:52:21.216962Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/sources.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/sources.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags.json new file mode 100644 index 0000000000..0745f3549b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 1202997, + "id": 62496068, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:8b8a5cef7bc416c719b7276affbd7555ae22d5721cb905e8d191fe00b0b951e3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 21943646, + "status": "stale", + "last_pulled": "2020-11-11T18:43:58.252311Z", + "last_pushed": "2020-09-29T20:52:20.774509Z" + } + ], + "last_updated": "2020-09-29T20:52:20.774509Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 7415504, + "full_size": 21943646, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-11T18:43:58.252311Z", + "tag_last_pushed": "2020-09-29T20:52:20.774509Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.images.json new file mode 100644 index 0000000000..67bb172938 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.images.json @@ -0,0 +1,40 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:8b8a5cef7bc416c719b7276affbd7555ae22d5721cb905e8d191fe00b0b951e3", + "layers": [ + { + "digest": "sha256:5d2415897100578ccbd3a0f03c205c3554ebe0ce4068a6969f7017d7dc0b1417", + "size": 2797239, + "instruction": "ADD file:a4192f640e3909ff3d4886840e28e983cb331b6d23219e6fb5a90471b7def94f in / " + }, + { + "size": 0, + "instruction": " CMD [\"/bin/sh\"]" + }, + { + "digest": "sha256:ca76a6be0a7e5963ec329fec39fa977888c6ea5292a600ce91223fb15f9176ca", + "size": 19145172, + "instruction": "/bin/sh -c apk upgrade --no-cache && apk add --no-cache rsync supervisor bash coreutils findutils" + }, + { + "digest": "sha256:c6ef8d11a87ba8cc65f39a8a7b51b60d52fd1a1aed1f6a24abf7f2a09b6dd904", + "size": 1235, + "instruction": "COPY dir:51b32cc696da2426e94bc769e0b5a75362f54388bacc9ab6ee15e4245cec4d3e in / " + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"supervisord\" \"-nkc/etc/supervisord.conf\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 21943646, + "status": "stale", + "last_pulled": "2020-11-11T18:43:58.252311Z", + "last_pushed": "2020-09-29T20:52:20.774509Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.info.json new file mode 100644 index 0000000000..8baad2de0b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/ateam-airsync/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 1202997, + "id": 62496068, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:8b8a5cef7bc416c719b7276affbd7555ae22d5721cb905e8d191fe00b0b951e3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 21943646, + "status": "stale", + "last_pulled": "2020-11-11T18:43:58.252311Z", + "last_pushed": "2020-09-29T20:52:20.774509Z" + } + ], + "last_updated": "2020-09-29T20:52:20.774509Z", + "last_updater": 1202997, + "last_updater_username": "fusl", + "name": "latest", + "repository": 7415504, + "full_size": 21943646, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-11T18:43:58.252311Z", + "tag_last_pushed": "2020-09-29T20:52:20.774509Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/Dockerfile new file mode 100644 index 0000000000..ec760688d1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM warcforceone/grab-base +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua \ + && chmod +x /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/info.json new file mode 100644 index 0000000000..e679a8cf48 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "bitbucket-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 976507, + "last_updated": "2020-10-15T14:32:38.301135Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "bitbucket-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Bitbucket](http://archiveteam.org/index.php?title=Bitbucket)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Bitbucket\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n pip install --upgrade seesaw zstandard\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/bitbucket-grab.git; cd bitbucket-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/bitbucket-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y install autoconf automake flex gnutls-devel lua-devel python-pip zlib-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, bitbucket-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/bitbucket-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/bitbucket-grab\n cd bitbucket-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #kickthebucket.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/sources.json new file mode 100644 index 0000000000..973b594101 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/8c7a76f2-4372-4093-a4c0-726c9ddce274/" + ], + "channel": "Stable", + "image": "warcforceone/bitbucket-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "bitbucket-grab", + "resource_uri": "/api/build/v1/source/4d235ec0-f970-4725-8029-205a4d1c3a37/", + "state": "Failed", + "uuid": "4d235ec0-f970-4725-8029-205a4d1c3a37" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags.json new file mode 100644 index 0000000000..62be68fed3 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 105714713, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:0205a9dfcd1de546e61e54216a0ea20b549831e605f2207d9fd25976dbe8914e", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368879851, + "status": "stale", + "last_pulled": "2021-01-12T17:24:17.88938Z", + "last_pushed": "2020-10-15T14:32:37.681063Z" + } + ], + "last_updated": "2020-10-15T14:32:37.681063Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9299518, + "full_size": 368879851, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-12T17:24:17.88938Z", + "tag_last_pushed": "2020-10-15T14:32:37.681063Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.images.json new file mode 100644 index 0000000000..529dd0a02f --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:0205a9dfcd1de546e61e54216a0ea20b549831e605f2207d9fd25976dbe8914e", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/fa7dc83944936bf09a0e4cb5d5ec852c0d256599/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6e0bb0a2c2533361d7f297ed547237caf1b7507f197835974c0dd7eba998c53c" + }, + { + "digest": "sha256:3eb2f327eb62905a17b993e9807eeb1c85f9c0172b0721b089586c9fdd4acaee", + "size": 2119480, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:0a47d90c3940a5dd80561f5659e27a5a7f8f7675d95420319ca74039ed94a671", + "size": 15164957, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b538559a01f82dea3b4dfad9ae58884da104ac09290b66f24874efe1a1dd458b", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:7a4090d986ab6ba276281b40ac7957d8206dcfc019a7df78a60086cca298403f", + "size": 14111449, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:c4a8fbe3409cd4f88f4e2671a6e590b3a6aeab77a9fb3c681d6ec758a9184450", + "size": 68666, + "instruction": "COPY dir:a1bb2ce0bcd0c1144c29cd12956892db1bfd40c9e0777f7515adbf0203e0ee88 in /grab " + }, + { + "digest": "sha256:ae5d978a3c8c3753c4c475547a48cbbcb53a3924133cdaa1e2e6f5d2b6fcfb24", + "size": 783695, + "instruction": "/bin/sh -c wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua && chmod +x /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 368879851, + "status": "stale", + "last_pulled": "2021-01-12T17:24:17.88938Z", + "last_pushed": "2020-10-15T14:32:37.681063Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.info.json new file mode 100644 index 0000000000..4b1f4dc39d --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/bitbucket-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 105714713, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:0205a9dfcd1de546e61e54216a0ea20b549831e605f2207d9fd25976dbe8914e", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368879851, + "status": "stale", + "last_pulled": "2021-01-12T17:24:17.88938Z", + "last_pushed": "2020-10-15T14:32:37.681063Z" + } + ], + "last_updated": "2020-10-15T14:32:37.681063Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9299518, + "full_size": 368879851, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-12T17:24:17.88938Z", + "tag_last_pushed": "2020-10-15T14:32:37.681063Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/Dockerfile new file mode 100644 index 0000000000..9b78304ce6 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/info.json new file mode 100644 index 0000000000..5c93942549 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "clutch-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11078089, + "last_updated": "2020-08-20T14:53:12.542994Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "clutch-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Clutch](http://archiveteam.org/index.php?title=Clutch)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Clutch\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/clutch-grab.git; cd clutch-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/clutch-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, clutch-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/clutch-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/clutch-grab\n cd clutch-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #pearls.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/sources.json new file mode 100644 index 0000000000..d2e21861bc --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/a199da12-3f64-4fb0-b8de-e6b53807afbe/" + ], + "channel": "Stable", + "image": "warcforceone/clutch-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "clutch-grab", + "resource_uri": "/api/build/v1/source/4e33a9d4-d56f-432e-b0d0-79591a919f20/", + "state": "Failed", + "uuid": "4e33a9d4-d56f-432e-b0d0-79591a919f20" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags.json new file mode 100644 index 0000000000..15f79e5f2a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 112683426, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:4e84f56d35afe3c9c98e579b5e21d4d8eaed315a1b4f961d2fc85476d8ed56ec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368321345, + "status": "active", + "last_pulled": "2021-02-09T18:14:02.880747Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-20T14:53:12.187023Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9612088, + "full_size": 368321345, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T18:14:02.880747Z", + "tag_last_pushed": "2020-08-20T14:53:12.187023Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.images.json new file mode 100644 index 0000000000..ab97205f76 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.images.json @@ -0,0 +1,130 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:4e84f56d35afe3c9c98e579b5e21d4d8eaed315a1b4f961d2fc85476d8ed56ec", + "layers": [ + { + "digest": "sha256:54f7e8ac135a5f502a6ee9537ef3d64b1cd2fa570dc0a40b4d3b6f7ac81e7486", + "size": 45320257, + "instruction": "ADD file:a61c14b18252183a4719980da97ac483044bcaa9df8a569a6d7bf0f719384b5e in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:d6341e30912f12f56e18564a3b582853f65376766f5f9d641a68a724ed6db88f", + "size": 10740278, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:087a57faf9491b1b82a83e26bc8cc90c90c30e4a4d858b57ddd5b4c2c90095f6", + "size": 4340083, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:5d71636fb824265e30ff34bf20737c9cdc4f5af28b6bce86f08215c55b89bfab", + "size": 50063138, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tbzr \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0c1db95989906f161007d8ef2a6ef6e0ec64bc15bf2c993fd002edbdfc7aa7df", + "size": 213213743, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibgeoip-dev \t\tlibglib2.0-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:bfb904e99f247ef4b354eab1285ec78bc5bb929dbf134ff69c156b90711216f6", + "size": 5830951, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.7.1" + }, + { + "digest": "sha256:78a3d3a96a32d4a5283b45966b9c49576fd5914f47797170104043068e3d87b0", + "size": 22366555, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& ldconfig \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:885a0ed92c894cb8a2abfc207d6a17cade6c8caea91ffe184e47cf3de08ed278", + "size": 240, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=18.1" + }, + { + "digest": "sha256:dd7cc9ace2427e64037752916207e90ff203948d45fff8eb41cd250413c27f10", + "size": 1783813, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/W9UAWrao_ftJLSVoBXf3EK9YLffbBUjv/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:183c3e8ba83dfffeba837534f42a22ebe6f2ecec0328f7794a63e452a0e39b40", + "size": 14606962, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/W9UAWrao_ftJLSVoBXf3EK9YLffbBUjv/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:34c26af70ca9b8455f86e0765625d8f2baaaef7c00e9e26589e824501d5eb13c", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:8f3108b81aa1e5ce457a37d90ec07b0bb9884285cf705aa0b21f32db87965001", + "size": 55055, + "instruction": "COPY dir:fc6adad130088938c606963c4a9738d8fd20acd154ea1fc6523380f8c022d506 in /grab " + }, + { + "digest": "sha256:216f9e32199814ec322fcb0964509f80d0e9d681f7892e4fc93ed7164d1c15da", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 368321345, + "status": "active", + "last_pulled": "2021-02-09T18:14:02.880747Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.info.json new file mode 100644 index 0000000000..7f3dd130ef --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutch-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 112683426, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:4e84f56d35afe3c9c98e579b5e21d4d8eaed315a1b4f961d2fc85476d8ed56ec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368321345, + "status": "active", + "last_pulled": "2021-02-09T18:14:02.880747Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-20T14:53:12.187023Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9612088, + "full_size": 368321345, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T18:14:02.880747Z", + "tag_last_pushed": "2020-08-20T14:53:12.187023Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/Dockerfile new file mode 100644 index 0000000000..9b78304ce6 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/info.json new file mode 100644 index 0000000000..0f6c5cda5b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "clutchvideos-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 6455998, + "last_updated": "2020-08-15T01:12:03.411902Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "clutchvideos-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Clutch](http://archiveteam.org/index.php?title=Clutch)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Clutch\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/clutchvideos-grab.git; cd clutchvideos-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/clutchvideos-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, clutchvideos-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/clutchvideos-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/clutchvideos-grab\n cd clutchvideos-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #pearls.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/sources.json new file mode 100644 index 0000000000..b7faeaef11 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/26959ddf-5953-4cab-ba7d-bdeaaeef621f/" + ], + "channel": "Stable", + "image": "warcforceone/clutchvideos-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "clutchvideos-grab", + "resource_uri": "/api/build/v1/source/34096de8-6194-4e01-a504-35a76d008e7e/", + "state": "Success", + "uuid": "34096de8-6194-4e01-a504-35a76d008e7e" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags.json new file mode 100644 index 0000000000..5ab1ae30a2 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 112845071, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:37ea72da5764da42fc820e8d1138daa9a27abdd8c3c2ba66484d6ec825be4fc9", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368321201, + "status": "stale", + "last_pulled": "2020-10-07T08:05:02.804907Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-15T01:12:03.104979Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9619291, + "full_size": 368321201, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-07T08:05:02.804907Z", + "tag_last_pushed": "2020-08-15T01:12:03.104979Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.images.json new file mode 100644 index 0000000000..ade936dab4 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.images.json @@ -0,0 +1,130 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:37ea72da5764da42fc820e8d1138daa9a27abdd8c3c2ba66484d6ec825be4fc9", + "layers": [ + { + "digest": "sha256:54f7e8ac135a5f502a6ee9537ef3d64b1cd2fa570dc0a40b4d3b6f7ac81e7486", + "size": 45320257, + "instruction": "ADD file:a61c14b18252183a4719980da97ac483044bcaa9df8a569a6d7bf0f719384b5e in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:d6341e30912f12f56e18564a3b582853f65376766f5f9d641a68a724ed6db88f", + "size": 10740278, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:087a57faf9491b1b82a83e26bc8cc90c90c30e4a4d858b57ddd5b4c2c90095f6", + "size": 4340083, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:5d71636fb824265e30ff34bf20737c9cdc4f5af28b6bce86f08215c55b89bfab", + "size": 50063138, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tbzr \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0c1db95989906f161007d8ef2a6ef6e0ec64bc15bf2c993fd002edbdfc7aa7df", + "size": 213213743, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibgeoip-dev \t\tlibglib2.0-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:bfb904e99f247ef4b354eab1285ec78bc5bb929dbf134ff69c156b90711216f6", + "size": 5830951, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.7.1" + }, + { + "digest": "sha256:78a3d3a96a32d4a5283b45966b9c49576fd5914f47797170104043068e3d87b0", + "size": 22366555, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& ldconfig \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:885a0ed92c894cb8a2abfc207d6a17cade6c8caea91ffe184e47cf3de08ed278", + "size": 240, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=18.1" + }, + { + "digest": "sha256:dd7cc9ace2427e64037752916207e90ff203948d45fff8eb41cd250413c27f10", + "size": 1783813, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/W9UAWrao_ftJLSVoBXf3EK9YLffbBUjv/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:183c3e8ba83dfffeba837534f42a22ebe6f2ecec0328f7794a63e452a0e39b40", + "size": 14606962, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/W9UAWrao_ftJLSVoBXf3EK9YLffbBUjv/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:34c26af70ca9b8455f86e0765625d8f2baaaef7c00e9e26589e824501d5eb13c", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:e4ef773fecb2fd3112a76347a69ed884da202b16f595ee3a6faa44c13bb891ac", + "size": 54911, + "instruction": "COPY dir:5c75f82f8290bdca05cb08593b9abe57d4cfeeaf3412202ce3b5949366b43094 in /grab " + }, + { + "digest": "sha256:70fd27e6a0f97dd5bf7b46cc61d780cead60b1b3591795e1791a6d1b240b2001", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 368321201, + "status": "stale", + "last_pulled": "2020-10-07T08:05:02.804907Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.info.json new file mode 100644 index 0000000000..3867a85bd6 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/clutchvideos-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 112845071, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:37ea72da5764da42fc820e8d1138daa9a27abdd8c3c2ba66484d6ec825be4fc9", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 368321201, + "status": "stale", + "last_pulled": "2020-10-07T08:05:02.804907Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-15T01:12:03.104979Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9619291, + "full_size": 368321201, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-07T08:05:02.804907Z", + "tag_last_pushed": "2020-08-15T01:12:03.104979Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/Dockerfile new file mode 100644 index 0000000000..9f3938345b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:edge +RUN apk upgrade --no-cache \ + && apk add --no-cache dnsmasq +COPY dnsmasq.conf /etc/dnsmasq.conf +STOPSIGNAL SIGKILL +ENTRYPOINT ["dnsmasq", "-kC/etc/dnsmasq.conf"] diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/info.json new file mode 100644 index 0000000000..9496e04eb1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "dnsmasq", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 21109092, + "last_updated": "2021-02-20T00:18:15.003220Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/sources.json new file mode 100644 index 0000000000..db98452087 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/5a9868ba-14b5-41f4-b5ec-63fb2d1fa8a5/" + ], + "channel": "Stable", + "image": "warcforceone/dnsmasq", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "dnsmasq-df", + "resource_uri": "/api/build/v1/source/f7fc079c-ba63-4734-a23e-1d1f090fcd5d/", + "state": "Success", + "uuid": "f7fc079c-ba63-4734-a23e-1d1f090fcd5d" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags.json new file mode 100644 index 0000000000..3e2dad0142 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 103588681, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:18d8db27db7e148e707eb8ee5686fddb32704a9165b35ef438973d14524cd3c5", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 4477304, + "status": "active", + "last_pulled": "2021-03-03T01:56:51.578645Z", + "last_pushed": "2021-02-20T00:18:14.704711Z" + } + ], + "last_updated": "2021-02-20T00:18:14.704711Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9191572, + "full_size": 4477304, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T01:56:51.578645Z", + "tag_last_pushed": "2021-02-20T00:18:14.704711Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.images.json new file mode 100644 index 0000000000..31480e6ee2 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.images.json @@ -0,0 +1,44 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:18d8db27db7e148e707eb8ee5686fddb32704a9165b35ef438973d14524cd3c5", + "layers": [ + { + "digest": "sha256:fa704576706331d0bbffe493842fe09e56a6cab3aa869749cf873dc431910583", + "size": 2813556, + "instruction": "ADD file:a5c7d53b28a7eec81df661fdab57f00eca5edb2a1acef7726153777cb37e4bf5 in / " + }, + { + "size": 0, + "instruction": " CMD [\"/bin/sh\"]" + }, + { + "digest": "sha256:85cc12a3c308b6ffd8e9bed72a3fa706df87c27172e83232c796b9c0a74b2895", + "size": 1663458, + "instruction": "/bin/sh -c apk upgrade --no-cache && apk add --no-cache dnsmasq" + }, + { + "digest": "sha256:4f937faedb43a4374fa7785cdaa56c2119a1a24aa2fa340168a99a31f7df91af", + "size": 290, + "instruction": "COPY file:e87e66fc03be3f8487a53c1b432ef3465c6c58f4b8b1570121dee97ba046b2b9 in /etc/dnsmasq.conf " + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGKILL" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"dnsmasq\" \"-kC/etc/dnsmasq.conf\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 4477304, + "status": "active", + "last_pulled": "2021-03-03T01:56:51.578645Z", + "last_pushed": "2021-02-20T00:18:14.704711Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.info.json new file mode 100644 index 0000000000..7af65c751c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/dnsmasq/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 103588681, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:18d8db27db7e148e707eb8ee5686fddb32704a9165b35ef438973d14524cd3c5", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 4477304, + "status": "active", + "last_pulled": "2021-03-03T01:56:51.578645Z", + "last_pushed": "2021-02-20T00:18:14.704711Z" + } + ], + "last_updated": "2021-02-20T00:18:14.704711Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9191572, + "full_size": 4477304, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T01:56:51.578645Z", + "tag_last_pushed": "2021-02-20T00:18:14.704711Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/Dockerfile new file mode 100644 index 0000000000..926ace5f79 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/Dockerfile @@ -0,0 +1,4 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at +RUN sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python*/site-packages/seesaw/tracker.py diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/info.json new file mode 100644 index 0000000000..266382704b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "domains-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 151168, + "last_updated": "2021-01-09T14:53:49.620078Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "domains-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [domains](http://archiveteam.org/index.php?title=domains)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"domains\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests warcio\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/domains-grab.git; cd domains-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/domains-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, domains-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/domains-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/domains-grab\n cd domains-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.hackint.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC [#archiveteam-bs](https://webirc.hackint.org/#irc://irc.hackint.org/#archiveteam-bs).\n\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/sources.json new file mode 100644 index 0000000000..8ec658856f --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": false, + "build_settings": [ + "/api/build/v1/setting/9120e450-9874-441f-be56-bd7a56da2477/" + ], + "channel": "Stable", + "image": "warcforceone/domains-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "domains-grab", + "resource_uri": "/api/build/v1/source/a54eca2b-941a-4842-98b2-8ebca1c88850/", + "state": "Success", + "uuid": "a54eca2b-941a-4842-98b2-8ebca1c88850" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags.json new file mode 100644 index 0000000000..e5d22712f1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 131529596, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:7636ab505a2f7313288afe1f3f362743fe7273d87e2d6e9432e6af92dcc70290", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355240806, + "status": "active", + "last_pulled": "2021-02-20T02:22:04.585308Z", + "last_pushed": "2021-01-09T14:53:43.504044Z" + } + ], + "last_updated": "2021-01-09T14:53:43.504044Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 10571662, + "full_size": 355240806, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:04.585308Z", + "tag_last_pushed": "2021-01-09T14:53:43.504044Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.images.json new file mode 100644 index 0000000000..9dc83c99c5 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.images.json @@ -0,0 +1,159 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:7636ab505a2f7313288afe1f3f362743fe7273d87e2d6e9432e6af92dcc70290", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:b401f610667fcd2db4f2c208848d0cf92f2ad490514b5443b3939a45d8b64b4d", + "size": 40895, + "instruction": "COPY dir:42f701425aa6570b0795acaecde911804d3f1da4686513072e0caba2726eeef2 in /grab " + }, + { + "digest": "sha256:59a0759a68a7e6a31ea6fc1f6d559162a678ad5ed66e38b4fe4c315b79ad409b", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + }, + { + "digest": "sha256:27395fd4e260c6ddae6954c3d0ebb1f0e3bd2ee6c3577d949efdaae714ed892b", + "size": 2894, + "instruction": "/bin/sh -c sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python*/site-packages/seesaw/tracker.py" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355240806, + "status": "active", + "last_pulled": "2021-02-20T02:22:04.585308Z", + "last_pushed": "2021-01-09T14:53:43.504044Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.info.json new file mode 100644 index 0000000000..b9265a0576 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/domains-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 131529596, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:7636ab505a2f7313288afe1f3f362743fe7273d87e2d6e9432e6af92dcc70290", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355240806, + "status": "active", + "last_pulled": "2021-02-20T02:22:04.585308Z", + "last_pushed": "2021-01-09T14:53:43.504044Z" + } + ], + "last_updated": "2021-01-09T14:53:43.504044Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 10571662, + "full_size": 355240806, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:04.585308Z", + "tag_last_pushed": "2021-01-09T14:53:43.504044Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/info.json new file mode 100644 index 0000000000..84a0e29162 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "fotoalbum-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 835123, + "last_updated": "2020-10-31T11:11:27.569458Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "fotoalbum-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Fotoalbum](http://archiveteam.org/index.php?title=Fotoalbum)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Fotoalbum\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/fotoalbum-grab.git; cd fotoalbum-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/fotoalbum-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, fotoalbum-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/fotoalbum-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/fotoalbum-grab\n cd fotoalbum-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #lookatthisfotograph.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/sources.json new file mode 100644 index 0000000000..00142a1706 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": false, + "build_settings": [ + "/api/build/v1/setting/5385947a-fd11-4d67-9bfc-e449ad1c530f/" + ], + "channel": "Stable", + "image": "warcforceone/fotoalbum-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "fotoalbum-grab", + "resource_uri": "/api/build/v1/source/9e8acc0f-7d65-4c36-a9f5-2b2f5678a33f/", + "state": "Success", + "uuid": "9e8acc0f-7d65-4c36-a9f5-2b2f5678a33f" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags.json new file mode 100644 index 0000000000..fd32922b58 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 119428392, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:07279a1bdd8c84d98c0db0fbc6b03f7c2819e4d049f1c7bcea1c59e36770ea9b", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356537761, + "status": "stale", + "last_pulled": "2021-01-25T09:00:20.61391Z", + "last_pushed": "2020-10-31T11:11:27.176579Z" + } + ], + "last_updated": "2020-10-31T11:11:27.176579Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9933212, + "full_size": 356537761, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-25T09:00:20.61391Z", + "tag_last_pushed": "2020-10-31T11:11:27.176579Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.images.json new file mode 100644 index 0000000000..bc4dbabd05 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:07279a1bdd8c84d98c0db0fbc6b03f7c2819e4d049f1c7bcea1c59e36770ea9b", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.0" + }, + { + "digest": "sha256:b1d09d0eabcb6f4e10ef0d360714bc5c780ea017bbcdc1a7e1a4a617ff8fb431", + "size": 20579241, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:475299e7c7f3a160843a7c349c788b8c9ec2ce9dd3b924e42830a38995955c9d", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:d2fe14d8e6bcb0ddb14c3ae2bc50b0f9cad6f42b33935609d160555f3525d71e", + "size": 2120746, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:76cccca5d2434bc713235d52a22280085ecc46e033071abae779166dd01288d7", + "size": 15354788, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:23ae9b84d8c637d7be11896ab80072718fff61a7f7779bafdab94a2e1a944695", + "size": 125, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-10-31T10:57:28Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=0865fe5498c6ef4730c784dd9c6a4f4e1f98a4f3" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:97aac8ae4b3219829025132fbd5add15c9560969fb45221f8d0b0af41e788973", + "size": 40124, + "instruction": "COPY dir:363525162a852e652aebbd7e979aa8cdd809532345a582ab56f91e419baaa320 in /grab " + }, + { + "digest": "sha256:bd702c4a8b5cbc85184adfe8a8c26b0ce775a8fceefe7b442258fcde20059764", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 356537761, + "status": "stale", + "last_pulled": "2021-01-25T09:00:20.61391Z", + "last_pushed": "2020-10-31T11:11:27.176579Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.info.json new file mode 100644 index 0000000000..b43c80cc91 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/fotoalbum-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 119428392, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:07279a1bdd8c84d98c0db0fbc6b03f7c2819e4d049f1c7bcea1c59e36770ea9b", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356537761, + "status": "stale", + "last_pulled": "2021-01-25T09:00:20.61391Z", + "last_pushed": "2020-10-31T11:11:27.176579Z" + } + ], + "last_updated": "2020-10-31T11:11:27.176579Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9933212, + "full_size": 356537761, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-25T09:00:20.61391Z", + "tag_last_pushed": "2020-10-31T11:11:27.176579Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/info.json new file mode 100644 index 0000000000..ba5b9fa2c0 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "github-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 13992747, + "last_updated": "2021-02-03T13:48:21.932838Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "github-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [GitHub](http://archiveteam.org/index.php?title=GitHub)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"GitHub\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/github-grab.git; cd github-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/github-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, github-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/github-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/github-grab\n cd github-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #gitgud.\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/sources.json new file mode 100644 index 0000000000..40a717223a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/77062f2a-f999-49ce-b7da-81cddd14bcb9/" + ], + "channel": "Stable", + "image": "warcforceone/github-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "github-grab", + "resource_uri": "/api/build/v1/source/81d70921-952f-4507-97c7-7a3c761ee3c7/", + "state": "Success", + "uuid": "81d70921-952f-4507-97c7-7a3c761ee3c7" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags.json new file mode 100644 index 0000000000..055b35add7 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 112047604, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2e3c90259794dee5979c9745ac9c73ef3ea5fb639cc4e745e1bcfc1854a7787a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355285549, + "status": "active", + "last_pulled": "2021-03-03T01:34:55.629655Z", + "last_pushed": "2021-02-03T13:48:21.641084Z" + } + ], + "last_updated": "2021-02-03T13:48:21.641084Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9584009, + "full_size": 355285549, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T01:34:55.629655Z", + "tag_last_pushed": "2021-02-03T13:48:21.641084Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.images.json new file mode 100644 index 0000000000..e1e6cd202c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:2e3c90259794dee5979c9745ac9c73ef3ea5fb639cc4e745e1bcfc1854a7787a", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:b25abe0ecc0925ce877cb7b359bdc9edd6e32fc8dde1adcc072a8c4883a15421", + "size": 88533, + "instruction": "COPY dir:84ec40390e24b81471934710a181eca724e76575a723f9297b3c39a1275afc17 in /grab " + }, + { + "digest": "sha256:dbdfcbfa61388a66f82bfefecec90f757fee49684cf94bc47bcbacb4e113f6c2", + "size": 142, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355285549, + "status": "active", + "last_pulled": "2021-03-03T01:34:55.629655Z", + "last_pushed": "2021-02-03T13:48:21.641084Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.info.json new file mode 100644 index 0000000000..3aa6b6b239 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/github-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 112047604, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2e3c90259794dee5979c9745ac9c73ef3ea5fb639cc4e745e1bcfc1854a7787a", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355285549, + "status": "active", + "last_pulled": "2021-03-03T01:34:55.629655Z", + "last_pushed": "2021-02-03T13:48:21.641084Z" + } + ], + "last_updated": "2021-02-03T13:48:21.641084Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9584009, + "full_size": 355285549, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T01:34:55.629655Z", + "tag_last_pushed": "2021-02-03T13:48:21.641084Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/info.json new file mode 100644 index 0000000000..5379d57903 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "google-sites-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7181229, + "last_updated": "2021-02-03T14:12:19.864923Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "google-sites-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Google Sites](http://archiveteam.org/index.php?title=Google_Sites)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Google Sites\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/google-sites-grab.git; cd google-sites-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/google-sites-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, google-sites-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/google-sites-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/google-sites-grab\n cd google-sites-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint channel #nearlylostmygoogles.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/sources.json new file mode 100644 index 0000000000..48c3966b5b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/9a200b7e-6ed7-4e5a-8048-094c5fb2458c/" + ], + "channel": "Stable", + "image": "warcforceone/google-sites-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "google-sites-grab", + "resource_uri": "/api/build/v1/source/18835edd-03c6-4e7e-ad53-61fca2dd7c8d/", + "state": "Success", + "uuid": "18835edd-03c6-4e7e-ad53-61fca2dd7c8d" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags.json new file mode 100644 index 0000000000..7b36b1dad3 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 120283946, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:ca02cf2b332f49cc3dc48c4795fbf9ea0f0ada8181e8a1f2ad3879101890b276", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355260303, + "status": "active", + "last_pulled": "2021-03-02T14:31:37.525395Z", + "last_pushed": "2021-02-03T14:12:19.517114Z" + } + ], + "last_updated": "2021-02-03T14:12:19.517114Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9974747, + "full_size": 355260303, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-02T14:31:37.525395Z", + "tag_last_pushed": "2021-02-03T14:12:19.517114Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.images.json new file mode 100644 index 0000000000..6b029841e4 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:ca02cf2b332f49cc3dc48c4795fbf9ea0f0ada8181e8a1f2ad3879101890b276", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:099b59f5dd443f225ff2ccf9f8fd5c62b1196ed3b76d4371c51228be4403b223", + "size": 63286, + "instruction": "COPY dir:056e2a67952ab9f3bad6a329f21ab48ab632b7b007755049a4c9e507abe9b3d1 in /grab " + }, + { + "digest": "sha256:acf1a14524374193f5d8b9e87eaafa8e56994dda04ce72a914147c67ca0a54b1", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355260303, + "status": "active", + "last_pulled": "2021-03-02T14:31:37.525395Z", + "last_pushed": "2021-02-03T14:12:19.517114Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.info.json new file mode 100644 index 0000000000..0addc64cde --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/google-sites-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 120283946, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:ca02cf2b332f49cc3dc48c4795fbf9ea0f0ada8181e8a1f2ad3879101890b276", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355260303, + "status": "active", + "last_pulled": "2021-03-02T14:31:37.525395Z", + "last_pushed": "2021-02-03T14:12:19.517114Z" + } + ], + "last_updated": "2021-02-03T14:12:19.517114Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9974747, + "full_size": 355260303, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-02T14:31:37.525395Z", + "tag_last_pushed": "2021-02-03T14:12:19.517114Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/Dockerfile new file mode 100644 index 0000000000..5691e6858a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/Dockerfile @@ -0,0 +1,15 @@ +ARG TLSTYPE=openssl +FROM atdr.meo.ws/archiveteam/wget-lua:v1.20.3-at-${TLSTYPE} AS wget +FROM python:3-slim +COPY --from=wget /wget /usr/local/bin/wget-lua +ENV LC_ALL=C +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket libpsl5 git \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install requests seesaw warcio zstandard \ + && chmod +x /usr/local/bin/wget-lua \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /grab +STOPSIGNAL SIGINT +ENTRYPOINT ["run-pipeline3", "--disable-web-server", "pipeline.py"] diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/info.json new file mode 100644 index 0000000000..480a4da221 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "grab-base", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 50530, + "last_updated": "2021-02-13T11:25:52.346081Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "# grab-base-df\nBase Dockerfile for warrior project grab scripts\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/sources.json new file mode 100644 index 0000000000..ee8a0328b2 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/568fc7d1-559a-40a5-a27f-0706214a48b1/" + ], + "channel": "Stable", + "image": "warcforceone/grab-base", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "grab-base-df", + "resource_uri": "/api/build/v1/source/97b9831e-a2db-4298-ab9d-dabf34a171b3/", + "state": "Success", + "uuid": "97b9831e-a2db-4298-ab9d-dabf34a171b3" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags.json new file mode 100644 index 0000000000..a9b072eadc --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 63789861, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:269f12c7c0d41372eeab2b2a37e19f8dbf26a20328d1bd943b5be0b260a790a1", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 85903786, + "status": "active", + "last_pulled": "2021-02-20T02:21:58.476745Z", + "last_pushed": "2021-02-13T11:25:52.045638Z" + } + ], + "last_updated": "2021-02-13T11:25:52.045638Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470894, + "full_size": 85903786, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:58.476745Z", + "tag_last_pushed": "2021-02-13T11:25:52.045638Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.images.json new file mode 100644 index 0000000000..8938982bd8 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.images.json @@ -0,0 +1,105 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:269f12c7c0d41372eeab2b2a37e19f8dbf26a20328d1bd943b5be0b260a790a1", + "layers": [ + { + "digest": "sha256:45b42c59be334ecda0daaa139b2f7d310e45c564c5f12263b1b8e68ec9e810ed", + "size": 27095142, + "instruction": "ADD file:d5c41bfaf15180481d8606f50799297e3f49b8a258c7c2cd988ab2bf0013272d in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:8cd3485318dba8e325f6b665105e10c51f4011cf2e2b4378ae2b92e104ff3566", + "size": 2769103, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tnetbase \t\ttzdata \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:b430e1981978aa8083832db9dc3f927e4336fbccf20f27a0419a5d31f78205b7", + "size": 10888464, + "instruction": "/bin/sh -c set -ex \t\t&& savedAptMark=\"$(apt-mark showmanual)\" \t&& apt-get update && apt-get install -y --no-install-recommends \t\tdpkg-dev \t\tgcc \t\tlibbluetooth-dev \t\tlibbz2-dev \t\tlibc6-dev \t\tlibexpat1-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tliblzma-dev \t\tlibncursesw5-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tmake \t\ttk-dev \t\tuuid-dev \t\twget \t\txz-utils \t\tzlib1g-dev \t\t$(command -v gpg > /dev/null || echo 'gnupg dirmngr') \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tLDFLAGS=\"-Wl,--strip-all\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& apt-mark auto '.*' > /dev/null \t&& apt-mark manual $savedAptMark \t&& find /usr/local -type f -executable -not \\( -name '*tkinter*' \\) -exec ldd '{}' ';' \t\t| awk '/=>/ { print $(NF-1) }' \t\t| sort -u \t\t| xargs -r dpkg-query --search \t\t| cut -d: -f1 \t\t| sort -u \t\t| xargs -r apt-mark manual \t&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \t&& rm -rf /var/lib/apt/lists/* \t\t&& python3 --version" + }, + { + "digest": "sha256:37d877dc41c386154662483278f5b856d8c34e200d571eb41cee562506fa5bfb", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=21.0.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/4be3fe44ad9dedc028629ed1497052d65d281b8e/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=8006625804f55e1bd99ad4214fd07082fee27a1c35945648a58f9087a714e9d4" + }, + { + "digest": "sha256:0b70d4ba14ef92ccaa7839657e6fca3422f09041c50a44011d6a2382d0c23fe8", + "size": 2452248, + "instruction": "/bin/sh -c set -ex; \t\tsavedAptMark=\"$(apt-mark showmanual)\"; \tapt-get update; \tapt-get install -y --no-install-recommends wget; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tapt-mark auto '.*' > /dev/null; \t[ -z \"$savedAptMark\" ] || apt-mark manual $savedAptMark; \tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \trm -rf /var/lib/apt/lists/*; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "digest": "sha256:2a5987f3c885013cfbe6ee2d5fa8bf80cbc61323e0ac8aa63ab2720faa2b9531", + "size": 926625, + "instruction": "COPY file:c7a381cd8311209dda148a7d15ef252c496da848907afaa18f8371d09648021e in /usr/local/bin/wget-lua " + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:4c4e90548af1cdbeb6d8ce5cde1c907ac0a4e48aed698d9a80b2afd3d3128111", + "size": 41771879, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket libpsl5 git && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0c0bb03728b9b8735afce4e3653c7761db07f3245cfa66a44bda45f8022184c8", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 85903786, + "status": "active", + "last_pulled": "2021-02-20T02:21:58.476745Z", + "last_pushed": "2021-02-13T11:25:52.045638Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.info.json new file mode 100644 index 0000000000..d732c8707a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/grab-base/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 63789861, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:269f12c7c0d41372eeab2b2a37e19f8dbf26a20328d1bd943b5be0b260a790a1", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 85903786, + "status": "active", + "last_pulled": "2021-02-20T02:21:58.476745Z", + "last_pushed": "2021-02-13T11:25:52.045638Z" + } + ], + "last_updated": "2021-02-13T11:25:52.045638Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7470894, + "full_size": 85903786, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:21:58.476745Z", + "tag_last_pushed": "2021-02-13T11:25:52.045638Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/Dockerfile new file mode 100644 index 0000000000..d12e82a3e1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:stretch-slim +RUN echo 'deb http://ftp.de.debian.org/debian buster-backports main' >> /etc/apt/sources.list +RUN DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install python python2.7 rsync git ca-certificates curl python-pip \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard && pip install requests +COPY * factory/ +RUN rm -rf /factory/megawarc && git clone https://github.com/archiveteam/megawarc.git /factory/megawarc +WORKDIR /factory +COPY docker-boot.sh / +RUN chmod +x /docker-boot.sh +ENTRYPOINT ["/docker-boot.sh"] diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/info.json new file mode 100644 index 0000000000..ed91904f57 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "megawarc-factory", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 80, + "last_updated": "2020-12-29T13:44:29.663201Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "Archive Team megawarc factory\n=============================\nSome scripts to bundle Archive Team uploads and upload them to Archive.org. Use at your own risk; the scripts will need per-project adjustment.\n\nThese scripts make batches of uploaded warc.gz files, combine them into megawarcs and upload them to their permanent home on Archive.org.\n\nThree processes work together to make this happen:\n\n1. The chunker\n--------------\nThe chunker moves uploaded warc.gz files from the upload directory to a batch directory. When this directory has grown to 50GB, the chunker begins a new directory and moves the completed directory to the packing queue.\n\nThere can only be one chunker per upload directory. Chunking doesn't take long, if the files are not moving to a different filesystem.\n\n2. The packer\n-------------\nThe packer monitors the packing queue. When the chunker brings a new directory, the packer removes the directory from the queue and starts converting it into a megawarc (using the megawarc utility). When that is done, the packer moves the megawarc to the upload queue and removes the original warc files.\n\nIf necessary, multiple packers can work the same queue. Packing involves lots of gzipping and takes some time.\n\n3. The uploader\n---------------\nThe uploader monitors the upload queue. When the packer brings a new megawarc, the uploader removes the megawarc from the queue and uploads it to Archive.org. If the upload is successful, the uploader removes the megawarc.\n\nIf necessary, multiple uploaders can work the same queue.\n\n4. The offloader\n---------------\nThe offloader monitors the upload queue. Instead of uploading to Archive.org, the megawarc will be sent to another host via rsync. This is useful when Archive.org has issues. \n\nThis can be used at the same time as the uploader without issues.\n\n\nFilesystems\n-----------\nFrom the chunker to the uploader, the chunks move through the system as timestamped directories, e.g., 20130401213900.) This timestamp will also be used in the name of the uploaded item on Archive.org. The queues are directories. Processes 'claim' a chunk by moving it from the queue directory to their working directory. This assumes that `mv` is an atomic operation.\n\nFor efficiency and to maintain the atomicity of `mv`, the filesystem of the directories is very important:\n\n1. The Rsync upload directory, the chunker working directory, the packing queue and that side of the packer's working directory should all be on the same filesystem. This ensures that the uploaded warc.gz files never move to a different file system.\n2. The megawarc side of the packer's working directory, the upload queue and the uploader's working directory should also share a filesystem.\n\nFilesystems 1 and 2 do not have to be the same.\n\n\nConfiguration\n-------------\nCreate a configuration file called `config.sh` and place it in the directory where you start the scripts. See the `config.example.sh` for more details.\n\n\nRunning\n-------\nRun the scripts in `screen`, `tmux` or something similar. `touch RUN` before you start the scripts. Use `rm RUN` to stop gracefully.\n\n* `./chunk-multiple` (run exactly one)\n* `./pack-multiple` (you may run more than one)\n* `./upload-multiple` (you may run more than one)\n* `./offload-multiple` (you may run more than one, can work in tandem with `upload-multiple`)\n\nUtility scripts:\n\n* `./du-all` will run `du -hs` in all queues\n\n\nScheduling priorities\n---------------------\nThe packing script will use all your I/O capacity. Consider using `nice` and `ionice` to run in at a lower priority, so it doesn't hinder your incoming Rsync or outgoing curl uploads.\n\n* `ionice -c 2 -n 6 nice -n 19 ./pack-multiple`\n\n\nRecovering from errors\n----------------------\nThe scripts are designed not to lose data. If a script dies, you can look in its working directory for in-progress items and move them back to the queue.\n\n\nRequirements\n------------\nThese scripts use Bash and Curl.\n\nYou should clone https://github.com/ArchiveTeam/megawarc to the `megawarc/` subdirectory of these scripts. The megawarc utility requires Python and Gzip.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/sources.json new file mode 100644 index 0000000000..a3169843fa --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/dc5abaaa-13e5-4b5a-952b-cf2d5054b6cb/" + ], + "channel": "Stable", + "image": "warcforceone/megawarc-factory", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "archiveteam-megawarc-factory", + "resource_uri": "/api/build/v1/source/42f121c3-bd58-4f01-bcd6-1563dac5cf4a/", + "state": "Success", + "uuid": "42f121c3-bd58-4f01-bcd6-1563dac5cf4a" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags.json new file mode 100644 index 0000000000..5cb4b4853c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 1202997, + "id": 62496302, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:34a7d52683683018fd803c10087e27b6f2b5a4c499bcf4bf600983b8f38e3f51", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 83378492, + "status": "stale", + "last_pulled": "2021-01-19T17:27:32.510273Z", + "last_pushed": "2020-12-29T13:44:29.105803Z" + } + ], + "last_updated": "2020-12-29T13:44:29.105803Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7415529, + "full_size": 83378492, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-19T17:27:32.510273Z", + "tag_last_pushed": "2020-12-29T13:44:29.105803Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.images.json new file mode 100644 index 0000000000..a5133a46a1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.images.json @@ -0,0 +1,64 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:34a7d52683683018fd803c10087e27b6f2b5a4c499bcf4bf600983b8f38e3f51", + "layers": [ + { + "digest": "sha256:e50c3c9ef5a201a24959788dcbc7ebf88d95c63e132a4d7396ce541127afd88e", + "size": 22527860, + "instruction": "ADD file:f03e68a10b84e2342cfffbb8cdec1117c7f5e5d0dd004072e84efb62cfdf157c in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:7835e0e8e88fdf51a188a95e2e5568fa8f5312cb64f0876caf424ee3a27f2049", + "size": 296, + "instruction": "/bin/sh -c echo 'deb http://ftp.de.debian.org/debian buster-backports main' >> /etc/apt/sources.list" + }, + { + "digest": "sha256:05082f7965abaec3bff308fbc6f539d4b720825f4370ac45de47457b588e4d35", + "size": 60447857, + "instruction": "/bin/sh -c DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install python python2.7 rsync git ca-certificates curl python-pip && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard && pip install requests" + }, + { + "digest": "sha256:77a7f73d7caba36c241f1baeb5a8cb5e42da767b1dad8e4d1073ff550165e3f0", + "size": 233941, + "instruction": "COPY multi:1c3cbf1e872449e49263d6a6ca9dc6d0aeac6034ec7b51cedcee86a674988d7d in factory/ " + }, + { + "digest": "sha256:bee00402a192e2d99dec3b0a9cf03a757fd0d2244d7ba265fff8ca1862ebcf26", + "size": 166800, + "instruction": "/bin/sh -c rm -rf /factory/megawarc && git clone https://github.com/archiveteam/megawarc.git /factory/megawarc" + }, + { + "size": 0, + "instruction": "WORKDIR /factory" + }, + { + "digest": "sha256:391913c444fc4ede6574dfec5fc56b465d5647609cd093282f339c32a8a3f126", + "size": 869, + "instruction": "COPY file:3b1d6271f1ce8e2709239da0ccf7e9ef0619e35964033fb297cac2918ccd8e0f in / " + }, + { + "digest": "sha256:391913c444fc4ede6574dfec5fc56b465d5647609cd093282f339c32a8a3f126", + "size": 869, + "instruction": "/bin/sh -c chmod +x /docker-boot.sh" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"/docker-boot.sh\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 83378492, + "status": "stale", + "last_pulled": "2021-01-19T17:27:32.510273Z", + "last_pushed": "2020-12-29T13:44:29.105803Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.info.json new file mode 100644 index 0000000000..c55c299454 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/megawarc-factory/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 1202997, + "id": 62496302, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:34a7d52683683018fd803c10087e27b6f2b5a4c499bcf4bf600983b8f38e3f51", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 83378492, + "status": "stale", + "last_pulled": "2021-01-19T17:27:32.510273Z", + "last_pushed": "2020-12-29T13:44:29.105803Z" + } + ], + "last_updated": "2020-12-29T13:44:29.105803Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7415529, + "full_size": 83378492, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2021-01-19T17:27:32.510273Z", + "tag_last_pushed": "2020-12-29T13:44:29.105803Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/Dockerfile new file mode 100644 index 0000000000..8ca159063d --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket luarocks \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +RUN luarocks install cgilua +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/info.json new file mode 100644 index 0000000000..76cfd0280c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "mercurial-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 1139185, + "last_updated": "2020-10-31T10:57:10.413100Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "mercurial-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Mercurial](http://archiveteam.org/index.php?title=Mercurial)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Mercurial\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync luarocks \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n pip install --upgrade seesaw zstandard\n luarocks install cgilua\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/mercurial-grab.git; cd mercurial-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/mercurial-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync luarocks \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For Fedora:\n\n dnf -y groupinstall \"Development Tools\"\n dnf -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, mercurial-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/mercurial-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/mercurial-grab\n cd mercurial-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/sources.json new file mode 100644 index 0000000000..6fb6b33c0e --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/a01bc9a2-31eb-4898-9e91-5b00aaf3422d/" + ], + "channel": "Stable", + "image": "warcforceone/mercurial-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "mercurial-grab", + "resource_uri": "/api/build/v1/source/bebe2412-9622-49d4-947e-d05034aac71a/", + "state": "Success", + "uuid": "bebe2412-9622-49d4-947e-d05034aac71a" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags.json new file mode 100644 index 0000000000..b380f76cc8 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 104901174, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:f2780378a7690b2371048e4eab8c1b636f59b50790a8280c87182c34dd289d3d", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369671957, + "status": "stale", + "last_pulled": "2020-11-13T00:16:22.577072Z", + "last_pushed": "2020-10-31T10:57:10.072246Z" + } + ], + "last_updated": "2020-10-31T10:57:10.072246Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9259478, + "full_size": 369671957, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-13T00:16:22.577072Z", + "tag_last_pushed": "2020-10-31T10:57:10.072246Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.images.json new file mode 100644 index 0000000000..fa2cf5e0e8 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.images.json @@ -0,0 +1,164 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:f2780378a7690b2371048e4eab8c1b636f59b50790a8280c87182c34dd289d3d", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:7f7c8879a275c860b98da629884e116f34e7531db5595b6d8c22653883506ed2", + "size": 2120572, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:4734814b4cdffab76ba2d4228048572e21d13069ad52a46881c48b0789026a79", + "size": 15176911, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:febbd2aa869b044d7f8a9fb316bfadde45653860dee26ec0473aebbe642c4535", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-10-31T00:11:54Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=474a18cb229913cca0bea8309fd791a1c8975512" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:6ab65b315a4128b4566967d14104bd3b750be539a6089355878964e41e0b7041", + "size": 15459068, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket luarocks && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:4ca2a36a88f3b03f66d1669bcd28e1c73e26b793e4b9764e9d32b737980d6da8", + "size": 224053, + "instruction": "/bin/sh -c luarocks install cgilua" + }, + { + "digest": "sha256:760a6e1318bb7e86a834208ffd999e52c6f34864a9bc1786b076ce868094e116", + "size": 59572, + "instruction": "COPY dir:1a8346118c0d2a503aa85164193700c53361dfbe743bce35d9dd4928719f6b72 in /grab " + }, + { + "digest": "sha256:576f93ae126c14bf9c0440782d51877cd603e1ce81bede111ac0ea584ddd6b94", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 369671957, + "status": "stale", + "last_pulled": "2020-11-13T00:16:22.577072Z", + "last_pushed": "2020-10-31T10:57:10.072246Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.info.json new file mode 100644 index 0000000000..97a848b72c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mercurial-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 104901174, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:f2780378a7690b2371048e4eab8c1b636f59b50790a8280c87182c34dd289d3d", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369671957, + "status": "stale", + "last_pulled": "2020-11-13T00:16:22.577072Z", + "last_pushed": "2020-10-31T10:57:10.072246Z" + } + ], + "last_updated": "2020-10-31T10:57:10.072246Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9259478, + "full_size": 369671957, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-11-13T00:16:22.577072Z", + "tag_last_pushed": "2020-10-31T10:57:10.072246Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/Dockerfile new file mode 100644 index 0000000000..ec760688d1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM warcforceone/grab-base +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua \ + && chmod +x /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/info.json new file mode 100644 index 0000000000..4a99f2d6c9 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "microsoft-download-center-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 7473694, + "last_updated": "2020-08-03T20:33:39.400223Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "microsoft-download-center-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Microsoft](http://archiveteam.org/index.php?title=Microsoft)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Microsoft\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/microsoft-download-center-grab.git; cd microsoft-download-center-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/microsoft-download-center-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, microsoft-download-center-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/microsoft-download-center-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/microsoft-download-center-grab\n cd microsoft-download-center-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam-bs.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/sources.json new file mode 100644 index 0000000000..14655e915f --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/9f5c8d10-fecb-49a3-9c6d-30fff6546274/" + ], + "channel": "Stable", + "image": "warcforceone/microsoft-download-center-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "microsoft-download-center-grab", + "resource_uri": "/api/build/v1/source/e9a87532-0631-4b5d-9f45-1b636f89d90b/", + "state": "Success", + "uuid": "e9a87532-0631-4b5d-9f45-1b636f89d90b" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags.json new file mode 100644 index 0000000000..2d2ec75959 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 111680411, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:3056c8345f1c7a6723026db6d00d33194e589fd62cd1d2f4bf052d84480a20cf", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 376003894, + "status": "stale", + "last_pulled": "2020-12-15T02:17:51.949718Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-03T20:33:39.101348Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9571009, + "full_size": 376003894, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-12-15T02:17:51.949718Z", + "tag_last_pushed": "2020-08-03T20:33:39.101348Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.images.json new file mode 100644 index 0000000000..c10156c929 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:3056c8345f1c7a6723026db6d00d33194e589fd62cd1d2f4bf052d84480a20cf", + "layers": [ + { + "digest": "sha256:5ae19949497e04289972756fe51cfac1a72b04fe2709e85a615945035c5a9a61", + "size": 50380042, + "instruction": "ADD file:2cddee716e84c40540a69c48051bd2dcf6cd3bd02a3e399334e97f20a77126ff in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ed3d96a2798e8837be24597cabf44ce25585cb9db1d749299cb06d51349ea5c2", + "size": 7804488, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f1213685078145f6136360475dbaffd0f86dfe92133a7bc26d79602980b255dd", + "size": 9978163, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:1a9ad5d5550bdff7db4c3d035bf9550bcd1de06a7f178a26de1d082591a5b956", + "size": 51765388, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:6f18049a0455d5e717b580354b515d9ac661e1a28d6ca9d6f7bc85c0dd17a7cf", + "size": 192264327, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibgeoip-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:ce39fa9d79d10239f5407e0bf43e2056f10cab5386d7319e220478fbed6a8323", + "size": 5792282, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.7.4" + }, + { + "digest": "sha256:3a91ffcf88eadccd4b196138e3c783070af7e5239688128d2c7ed400289574e4", + "size": 26568003, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tPROFILE_TASK='-m test.regrtest --pgo \t\t\ttest_array \t\t\ttest_base64 \t\t\ttest_binascii \t\t\ttest_binhex \t\t\ttest_binop \t\t\ttest_bytes \t\t\ttest_c_locale_coercion \t\t\ttest_class \t\t\ttest_cmath \t\t\ttest_codecs \t\t\ttest_compile \t\t\ttest_complex \t\t\ttest_csv \t\t\ttest_decimal \t\t\ttest_dict \t\t\ttest_float \t\t\ttest_fstring \t\t\ttest_hashlib \t\t\ttest_io \t\t\ttest_iter \t\t\ttest_json \t\t\ttest_long \t\t\ttest_math \t\t\ttest_memoryview \t\t\ttest_pickle \t\t\ttest_re \t\t\ttest_set \t\t\ttest_slice \t\t\ttest_struct \t\t\ttest_threading \t\t\ttest_time \t\t\ttest_traceback \t\t\ttest_unicode \t\t' \t&& make install \t&& ldconfig \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:ee82cc8e15068a2e57dec1805ebc11ae639bf10d967bf1b859794756916bd0de", + "size": 235, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=19.2.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/404c9418e33c5031b1a9ab623168b3e8a2ed8c88/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=56bb63d3cf54e7444351256f72a60f575f6d8c7f1faacffae33167afc8e7609d" + }, + { + "digest": "sha256:f0fe55e0f617741a0138eef3b2d1a300da7a877d9e53206d3c65d7328dec0a59", + "size": 1863155, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:1f7879d098d1a0827ae0c4e9cb37cb9904b08c29bc928204edd19184b551db56", + "size": 8241010, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua /bin/sh -c DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 && pip install requests seesaw warcio && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b20fd304fc72c42021381facad6cd033901da2d61f1ccc3bb54cd8f046ba9a49", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:b7c2709e8a3dd00cb69ef26124737c00c1cadf46c5df6b4191276bc8054d7150", + "size": 20523378, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:7876cf09513792530c5468080800b5c382700f8aabc233401f69a4437258d1c9", + "size": 39629, + "instruction": "COPY dir:0746463d343d8b6184800e143b4f8f236e3bf9c051a018af3329b1d045fcc9ad in /grab " + }, + { + "digest": "sha256:5241f8c8f2bdcb50675aa95fb8bb4071581e0a74a0320518e9388c5196825faf", + "size": 783701, + "instruction": "/bin/sh -c wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua && chmod +x /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 376003894, + "status": "stale", + "last_pulled": "2020-12-15T02:17:51.949718Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.info.json new file mode 100644 index 0000000000..94acd19d2b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/microsoft-download-center-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 111680411, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:3056c8345f1c7a6723026db6d00d33194e589fd62cd1d2f4bf052d84480a20cf", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 376003894, + "status": "stale", + "last_pulled": "2020-12-15T02:17:51.949718Z", + "last_pushed": null + } + ], + "last_updated": "2020-08-03T20:33:39.101348Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9571009, + "full_size": 376003894, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-12-15T02:17:51.949718Z", + "tag_last_pushed": "2020-08-03T20:33:39.101348Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/Dockerfile new file mode 100644 index 0000000000..ec760688d1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM warcforceone/grab-base +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua \ + && chmod +x /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/info.json new file mode 100644 index 0000000000..1cc32686b2 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "mixer-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 1299654, + "last_updated": "2020-07-23T17:26:35.210553Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "mixer-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Mixer](http://archiveteam.org/index.php?title=Mixer)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Mixer\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n pip install --upgrade seesaw zstandard\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/mixer-grab.git; cd mixer-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/mixer-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, mixer-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/mixer-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/mixer-grab\n cd mixer-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC #mixdown.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/sources.json new file mode 100644 index 0000000000..7e4451974f --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/686a7b02-8f15-46bc-b680-7edffbe25e94/" + ], + "channel": "Stable", + "image": "warcforceone/mixer-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "mixer-grab", + "resource_uri": "/api/build/v1/source/e43271ef-e1a0-4457-bb5c-68929e2f9f60/", + "state": "Success", + "uuid": "e43271ef-e1a0-4457-bb5c-68929e2f9f60" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags.json new file mode 100644 index 0000000000..e05bc933a8 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 109519505, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:6ac6ed3ba3e05eee234700d5d62088672fd7258408d3a4d0b6b525c90ea37476", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 376001986, + "status": "stale", + "last_pulled": "2020-10-26T21:57:05.240442Z", + "last_pushed": null + } + ], + "last_updated": "2020-07-23T17:26:34.913853Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9486064, + "full_size": 376001986, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-26T21:57:05.240442Z", + "tag_last_pushed": "2020-07-23T17:26:34.913853Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.images.json new file mode 100644 index 0000000000..80a87ace43 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:6ac6ed3ba3e05eee234700d5d62088672fd7258408d3a4d0b6b525c90ea37476", + "layers": [ + { + "digest": "sha256:5ae19949497e04289972756fe51cfac1a72b04fe2709e85a615945035c5a9a61", + "size": 50380042, + "instruction": "ADD file:2cddee716e84c40540a69c48051bd2dcf6cd3bd02a3e399334e97f20a77126ff in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ed3d96a2798e8837be24597cabf44ce25585cb9db1d749299cb06d51349ea5c2", + "size": 7804488, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f1213685078145f6136360475dbaffd0f86dfe92133a7bc26d79602980b255dd", + "size": 9978163, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:1a9ad5d5550bdff7db4c3d035bf9550bcd1de06a7f178a26de1d082591a5b956", + "size": 51765388, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:6f18049a0455d5e717b580354b515d9ac661e1a28d6ca9d6f7bc85c0dd17a7cf", + "size": 192264327, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibgeoip-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:ce39fa9d79d10239f5407e0bf43e2056f10cab5386d7319e220478fbed6a8323", + "size": 5792282, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.7.4" + }, + { + "digest": "sha256:3a91ffcf88eadccd4b196138e3c783070af7e5239688128d2c7ed400289574e4", + "size": 26568003, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tPROFILE_TASK='-m test.regrtest --pgo \t\t\ttest_array \t\t\ttest_base64 \t\t\ttest_binascii \t\t\ttest_binhex \t\t\ttest_binop \t\t\ttest_bytes \t\t\ttest_c_locale_coercion \t\t\ttest_class \t\t\ttest_cmath \t\t\ttest_codecs \t\t\ttest_compile \t\t\ttest_complex \t\t\ttest_csv \t\t\ttest_decimal \t\t\ttest_dict \t\t\ttest_float \t\t\ttest_fstring \t\t\ttest_hashlib \t\t\ttest_io \t\t\ttest_iter \t\t\ttest_json \t\t\ttest_long \t\t\ttest_math \t\t\ttest_memoryview \t\t\ttest_pickle \t\t\ttest_re \t\t\ttest_set \t\t\ttest_slice \t\t\ttest_struct \t\t\ttest_threading \t\t\ttest_time \t\t\ttest_traceback \t\t\ttest_unicode \t\t' \t&& make install \t&& ldconfig \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:ee82cc8e15068a2e57dec1805ebc11ae639bf10d967bf1b859794756916bd0de", + "size": 235, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=19.2.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/404c9418e33c5031b1a9ab623168b3e8a2ed8c88/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=56bb63d3cf54e7444351256f72a60f575f6d8c7f1faacffae33167afc8e7609d" + }, + { + "digest": "sha256:f0fe55e0f617741a0138eef3b2d1a300da7a877d9e53206d3c65d7328dec0a59", + "size": 1863155, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:1f7879d098d1a0827ae0c4e9cb37cb9904b08c29bc928204edd19184b551db56", + "size": 8241010, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua /bin/sh -c DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 && pip install requests seesaw warcio && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b20fd304fc72c42021381facad6cd033901da2d61f1ccc3bb54cd8f046ba9a49", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:e5ca08967acb00cc3e5dfd610da9b1b792ff7da003b98501a6fee19cfdcaac3f", + "size": 20498227, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:6b27e7eff5b63ffff0c956e2f5b607bc780f71628b643498b4350c7f1f475fcd", + "size": 62872, + "instruction": "COPY dir:816d8e423eaa2c3893ff83d274c94e20d55920ade73f1cde0b755d5d5f39512b in /grab " + }, + { + "digest": "sha256:3bedb2b0cbb09276688dbc05d8ba19e007f162dbe70eca777781d4cc42c3bb47", + "size": 783701, + "instruction": "/bin/sh -c wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua && chmod +x /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 376001986, + "status": "stale", + "last_pulled": "2020-10-26T21:57:05.240442Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.info.json new file mode 100644 index 0000000000..ff79d40e84 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/mixer-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 109519505, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:6ac6ed3ba3e05eee234700d5d62088672fd7258408d3a4d0b6b525c90ea37476", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 376001986, + "status": "stale", + "last_pulled": "2020-10-26T21:57:05.240442Z", + "last_pushed": null + } + ], + "last_updated": "2020-07-23T17:26:34.913853Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9486064, + "full_size": 376001986, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-26T21:57:05.240442Z", + "tag_last_pushed": "2020-07-23T17:26:34.913853Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/Dockerfile new file mode 100644 index 0000000000..9b78304ce6 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/info.json new file mode 100644 index 0000000000..4605a17d1d --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "naver-matome-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 8478031, + "last_updated": "2020-09-22T15:42:01.244574Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "naver-matome-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [NAVER\u307e\u3068\u3081](http://archiveteam.org/index.php?title=NAVER\u307e\u3068\u3081)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"NAVER\u307e\u3068\u3081\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/naver-matome-grab.git; cd naver-matome-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/naver-matome-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, naver-matome-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/naver-matome-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/naver-matome-grab\n cd naver-matome-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #navergonnagiveyouup.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/sources.json new file mode 100644 index 0000000000..3e63508462 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": false, + "build_settings": [ + "/api/build/v1/setting/5140fd44-a466-4220-8478-c38ea663b6d1/" + ], + "channel": "Stable", + "image": "warcforceone/naver-matome-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": false, + "repository": "naver-matome-grab", + "resource_uri": "/api/build/v1/source/f502765c-a143-4a78-8f59-8f85c6c7ae18/", + "state": "Success", + "uuid": "f502765c-a143-4a78-8f59-8f85c6c7ae18" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags.json new file mode 100644 index 0000000000..13e6b4521a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 118429736, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2bccc575ead8a3987dca7d692ee8bcc43c68b39dd5373d025affd70e91a7be44", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353764508, + "status": "active", + "last_pulled": "2021-02-09T20:45:19.121077Z", + "last_pushed": "2020-09-22T15:42:00.746382Z" + } + ], + "last_updated": "2020-09-22T15:42:00.746382Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9883515, + "full_size": 353764508, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T20:45:19.121077Z", + "tag_last_pushed": "2020-09-22T15:42:00.746382Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.images.json new file mode 100644 index 0000000000..dca12f21b9 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.images.json @@ -0,0 +1,138 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:2bccc575ead8a3987dca7d692ee8bcc43c68b39dd5373d025affd70e91a7be44", + "layers": [ + { + "digest": "sha256:57df1a1f1ad841deaf50c8f662d77e93b4b17af776ed66148116607f9aceffa8", + "size": 50395913, + "instruction": "ADD file:07a6578d6f507bd9c51bdf4fe41402db5dcf3b9fdf51cd4315778c27da1add39 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:71e126169501d71bbbd0d3c8d9f35836c41660869fe8432ac606341ed21f7adb", + "size": 7811567, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:1af28a55c3f320826db8df3146a2c198f9042877ef679f9e32210aa9a7fac9ef", + "size": 9996317, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:03f1c9932170e54fface2382b2550b8052ae3d41f27e66ea1294e2055dd2b2e7", + "size": 51829661, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:65b3db15f518f11e53c95664d0675a5d78a5329d18d5316a406c2a45907a0723", + "size": 192249513, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:3e3b8947ed83f2a39a813e0a9172cfdd571d4c943228a74e833af60e7536cac5", + "size": 6145377, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.5" + }, + { + "digest": "sha256:a4850b8bdbb7b624a1aaf5892b7409d5ae64f8dfdbb19e3017cf6e7fbbfb0ef9", + "size": 17998369, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:41653399496823f7855fe3bf2725fe0a1b79f2da59db9f3b8d5d9a5177c896ae", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/fa7dc83944936bf09a0e4cb5d5ec852c0d256599/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6e0bb0a2c2533361d7f297ed547237caf1b7507f197835974c0dd7eba998c53c" + }, + { + "digest": "sha256:1b580f9ce4cedf9f911da821c8445cd807bc29a7ab3cc62937b87604138dbfa8", + "size": 2119419, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:112d3c4c29c93c57de0b348a39deaadc7723662dc02032b81033038d1df819ec", + "size": 15159654, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:7c80e1fa56ef43f8e012bfb982788afe906a1e5cac2cc87dc307a1c376aa74f3", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:2dbac3ba6deca119b8c63326d2644d26ef1a27846fc42904f5e29214521afce1", + "size": 58249, + "instruction": "COPY dir:e7db9d6e2c7d9455fa5cb8a75918994482dc05b0d99915ccf7ba765a1637991e in /grab " + }, + { + "digest": "sha256:b9ae6b60d44890e1acbdaf9b444c52caf286d0130d829cb083264e2eeb3785fe", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 353764508, + "status": "active", + "last_pulled": "2021-02-09T20:45:19.121077Z", + "last_pushed": "2020-09-22T15:42:00.746382Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.info.json new file mode 100644 index 0000000000..f998b4ac54 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/naver-matome-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 118429736, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2bccc575ead8a3987dca7d692ee8bcc43c68b39dd5373d025affd70e91a7be44", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353764508, + "status": "active", + "last_pulled": "2021-02-09T20:45:19.121077Z", + "last_pushed": "2020-09-22T15:42:00.746382Z" + } + ], + "last_updated": "2020-09-22T15:42:00.746382Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9883515, + "full_size": 353764508, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-09T20:45:19.121077Z", + "tag_last_pushed": "2020-09-22T15:42:00.746382Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/info.json new file mode 100644 index 0000000000..ddb88e214a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "pastebin-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 386445, + "last_updated": "2021-02-03T13:42:13.640186Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "pastebin-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Pastebin](http://archiveteam.org/index.php?title=Pastebin)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Pastebin\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/pastebin-grab.git; cd pastebin-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/pastebin-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, pastebin-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/pastebin-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/pastebin-grab\n cd pastebin-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint #pastalavista.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/sources.json new file mode 100644 index 0000000000..2404f8f2f8 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/6cab4c66-a6bb-4239-9489-360d8cb03471/" + ], + "channel": "Stable", + "image": "warcforceone/pastebin-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "pastebin-grab", + "resource_uri": "/api/build/v1/source/7a5ac6cc-051c-4214-81a7-7eafc73cef7a/", + "state": "Success", + "uuid": "7a5ac6cc-051c-4214-81a7-7eafc73cef7a" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags.json new file mode 100644 index 0000000000..ce98cb0f9e --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 102735030, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:32e764fef061c155c249f947259a76f3e431c8236b4c2bd791d141c8e9fea873", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355261696, + "status": "active", + "last_pulled": "2021-02-24T08:11:59.226741Z", + "last_pushed": "2021-02-03T13:42:13.273216Z" + } + ], + "last_updated": "2021-02-03T13:42:13.273216Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9154879, + "full_size": 355261696, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-24T08:11:59.226741Z", + "tag_last_pushed": "2021-02-03T13:42:13.273216Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.images.json new file mode 100644 index 0000000000..fc5ab70240 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:32e764fef061c155c249f947259a76f3e431c8236b4c2bd791d141c8e9fea873", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:1536bb0c75b05971d8ca7e4d032f4f022286bef1f45d8158194d67faf1af1c07", + "size": 64679, + "instruction": "COPY dir:370d4a1793d1d393c73d1d4f418581af7db7b5e5e7a3e426ae221c8c5d03e5d1 in /grab " + }, + { + "digest": "sha256:ffaf42b9ffca673f5d75ceea50241c2c3b839d155888eb3c1f5cf62e622d36af", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355261696, + "status": "active", + "last_pulled": "2021-02-24T08:11:59.226741Z", + "last_pushed": "2021-02-03T13:42:13.273216Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.info.json new file mode 100644 index 0000000000..1d2d09171c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/pastebin-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 102735030, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:32e764fef061c155c249f947259a76f3e431c8236b4c2bd791d141c8e9fea873", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355261696, + "status": "active", + "last_pulled": "2021-02-24T08:11:59.226741Z", + "last_pushed": "2021-02-03T13:42:13.273216Z" + } + ], + "last_updated": "2021-02-03T13:42:13.273216Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9154879, + "full_size": 355261696, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-24T08:11:59.226741Z", + "tag_last_pushed": "2021-02-03T13:42:13.273216Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/Dockerfile new file mode 100644 index 0000000000..892855d054 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/Dockerfile @@ -0,0 +1,5 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -s /usr/local/bin/wget-lua /grab/wget-lua +# RUN wget -O /grab/wget-lua http://xor.meo.ws/vUO6LyuhBlMOqGUjZ3sFQCqUcR83pl9N/wget-lua \ +# && chmod +x /grab/wget-lua diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/info.json new file mode 100644 index 0000000000..c143e32bc1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "playstv-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 54289, + "last_updated": "2019-12-12T17:09:45.546166Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "playstv-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Plays.tv](http://archiveteam.org/index.php?title=Plays.tv)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Plays.tv\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n pip install --upgrade seesaw\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/playstv-grab.git; cd playstv-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/playstv-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n\nAnd you may also need to update your Python's warcio if you find screen detaching immediately after running the job in Ubuntu 18.04 Bionic and above:\n\n pip install warcio --upgrade\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y install autoconf automake flex gnutls-devel lua-devel python-pip zlib-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, playstv-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/playstv-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/playstv-grab\n cd playstv-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint.org #stops.tv.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/sources.json new file mode 100644 index 0000000000..a77a87a704 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/playstv-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/4978a4cd-6d53-4575-99c3-9a3295723583/" + ], + "channel": "Stable", + "image": "warcforceone/playstv-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "playstv-grab", + "resource_uri": "/api/build/v1/source/5aecf8e7-b72a-4ecc-ac3b-15f653e57c8f/", + "state": "Success", + "uuid": "5aecf8e7-b72a-4ecc-ac3b-15f653e57c8f" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/Dockerfile new file mode 100644 index 0000000000..4ddc498c00 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/Dockerfile @@ -0,0 +1,8 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/info.json new file mode 100644 index 0000000000..1e9313a13b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "reddit-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 5792737, + "last_updated": "2021-01-14T00:56:43.223341Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "reddit-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Reddit](http://archiveteam.org/index.php?title=Reddit)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Reddit\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/reddit-grab.git; cd reddit-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/reddit-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, reddit-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/reddit-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/reddit-grab\n cd reddit-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC #shreddit.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/sources.json new file mode 100644 index 0000000000..ef66da7c9b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/sources.json @@ -0,0 +1,16 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [], + "channel": "Stable", + "image": "warcforceone/reddit-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "reddit-grab", + "resource_uri": "/api/build/v1/source/dc7da00e-9940-41cc-adec-5993f2903fd1/", + "state": "Empty", + "uuid": "dc7da00e-9940-41cc-adec-5993f2903fd1" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags.json new file mode 100644 index 0000000000..b23b3d3350 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 110463951, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:861f767c36c60bc249fa1bb2c692c74eef87e3448e4179a7d72e027e0a3be802", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369971934, + "status": "active", + "last_pulled": "2021-02-04T14:13:47.006022Z", + "last_pushed": "2021-01-14T00:56:42.817606Z" + } + ], + "last_updated": "2021-01-14T00:56:42.817606Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9525328, + "full_size": 369971934, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-04T14:13:47.006022Z", + "tag_last_pushed": "2021-01-14T00:56:42.817606Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.images.json new file mode 100644 index 0000000000..376a253942 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.images.json @@ -0,0 +1,159 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:861f767c36c60bc249fa1bb2c692c74eef87e3448e4179a7d72e027e0a3be802", + "layers": [ + { + "digest": "sha256:6c33745f49b41daad28b7b192c447938452ea4de9fe8c7cc3edf1433b1366946", + "size": 50397728, + "instruction": "ADD file:6014cd9d7466825f80d4a3345847efd6fd7ef600b8135811cab4f0e304f66cd7 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ef072fc32a84ef237dd4fcc7dff2c5e2a77565f24d63977d0fa654a6d8512dd8", + "size": 7812075, + "instruction": "/bin/sh -c set -eux; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:c0afb8e68e0bcdc1b6e05acaa713a6fe0d818086c596bd1ad99133665c4efe63", + "size": 9996417, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:d599c07d28e6c920ef615f4f9b5cd0d52eb106fcd20c3a7daef389f14edd4ef5", + "size": 51829485, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f2ecc74db11a2e0bc5a12395b008dc12bdbaaeeb45da34f4fa7e75b3c17ed564", + "size": 192313866, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26856d31ce86a1119d0801da67e89e8cec83e0d53caca90a1cb8a9fa325b96a6", + "size": 6145411, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.1" + }, + { + "digest": "sha256:a463ae07b5f35ef689567ba040dda03e740798fa71eb1ecb55420baf6423b95c", + "size": 19137053, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:54f24c50f14e2b0fc059d866bbc32a75cd42858a79872b9ddec21f0d0f491dda", + "size": 232, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.3.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/5f38681f7f5872e4032860b54e9cc11cf0374932/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6a0b13826862f33c13b614a921d36253bfa1ae779c5fbf569876f3585057e9d2" + }, + { + "digest": "sha256:168ee6df05fe6efec9c36488270abba0fc69fb1530b419a5f32c02923e0a911f", + "size": 2138311, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:3ad64c2dd8aa09ba4c2abbb96e915b73906156c57f90f5dda3636ec8d08fe8e3", + "size": 15426169, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw warcio zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:4b9a241bf0391f375447a6ffd8270ec04ffc0f2c2ba768a54f757bad14e40184", + "size": 127, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-12-29T14:36:01Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=fd9edd4c19b94d1854ba787c3ddcabf3f9bfd1be" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:d5095accb4b2cc664998ce481eeceeb359fe52ffc449d4ea4923ef5b12890ec4", + "size": 14219511, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:56c04c083a6f74407eb93255b4338ab9aa75269bf33e60b44294f95a6a8adc90", + "size": 555407, + "instruction": "COPY dir:e0f48eaf4d219be53af5143432dbe6d5c9fa9dfdadbec36601e685f6a0c4b9cc in /grab " + }, + { + "digest": "sha256:eca96286643ebbbb9a297d8f16290d503c85efb44d98b0f1a82c69d582127403", + "size": 142, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 369971934, + "status": "active", + "last_pulled": "2021-02-04T14:13:47.006022Z", + "last_pushed": "2021-01-14T00:56:42.817606Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.info.json new file mode 100644 index 0000000000..ca171d7348 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/reddit-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 110463951, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:861f767c36c60bc249fa1bb2c692c74eef87e3448e4179a7d72e027e0a3be802", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369971934, + "status": "active", + "last_pulled": "2021-02-04T14:13:47.006022Z", + "last_pushed": "2021-01-14T00:56:42.817606Z" + } + ], + "last_updated": "2021-01-14T00:56:42.817606Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9525328, + "full_size": 369971934, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-04T14:13:47.006022Z", + "tag_last_pushed": "2021-01-14T00:56:42.817606Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/Dockerfile new file mode 100644 index 0000000000..892855d054 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/Dockerfile @@ -0,0 +1,5 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -s /usr/local/bin/wget-lua /grab/wget-lua +# RUN wget -O /grab/wget-lua http://xor.meo.ws/vUO6LyuhBlMOqGUjZ3sFQCqUcR83pl9N/wget-lua \ +# && chmod +x /grab/wget-lua diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/info.json new file mode 100644 index 0000000000..5e4acddafc --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "singstar-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 23644, + "last_updated": "2020-01-21T23:35:49.829575Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "singstar-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [SingStar](http://archiveteam.org/index.php?title=SingStar)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"SingStar\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n pip install --upgrade seesaw\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/singstar-grab.git; cd singstar-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/singstar-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n\nAnd you may also need to update your Python's warcio if you find screen detaching immediately after running the job in Ubuntu 18.04 Bionic and above:\n\n pip install warcio --upgrade\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y install autoconf automake flex gnutls-devel lua-devel python-pip zlib-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, singstar-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/singstar-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/singstar-grab\n cd singstar-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/sources.json new file mode 100644 index 0000000000..d90fc08e5a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/9747cb6e-3085-4810-aa89-78e5b595242d/" + ], + "channel": "Stable", + "image": "warcforceone/singstar-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "singstar-grab", + "resource_uri": "/api/build/v1/source/a7b49979-0504-4070-a22c-33bae282e6e9/", + "state": "Success", + "uuid": "a7b49979-0504-4070-a22c-33bae282e6e9" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags.json new file mode 100644 index 0000000000..c6aecfe226 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 83570324, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2631fb2f8a664f4e0a027018479146367f9d101011e6bb926b9c14e3bf8c1bb7", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355151353, + "status": "stale", + "last_pulled": "2020-10-09T05:52:28.669085Z", + "last_pushed": null + } + ], + "last_updated": "2020-01-21T23:35:49.453971Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 8355754, + "full_size": 355151353, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-09T05:52:28.669085Z", + "tag_last_pushed": "2020-01-21T23:35:49.453971Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.images.json new file mode 100644 index 0000000000..7dcd132302 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.images.json @@ -0,0 +1,138 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:2631fb2f8a664f4e0a027018479146367f9d101011e6bb926b9c14e3bf8c1bb7", + "layers": [ + { + "digest": "sha256:5ae19949497e04289972756fe51cfac1a72b04fe2709e85a615945035c5a9a61", + "size": 50380042, + "instruction": "ADD file:2cddee716e84c40540a69c48051bd2dcf6cd3bd02a3e399334e97f20a77126ff in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ed3d96a2798e8837be24597cabf44ce25585cb9db1d749299cb06d51349ea5c2", + "size": 7804488, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f1213685078145f6136360475dbaffd0f86dfe92133a7bc26d79602980b255dd", + "size": 9978163, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:1a9ad5d5550bdff7db4c3d035bf9550bcd1de06a7f178a26de1d082591a5b956", + "size": 51765388, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:6f18049a0455d5e717b580354b515d9ac661e1a28d6ca9d6f7bc85c0dd17a7cf", + "size": 192264327, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibgeoip-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:ce39fa9d79d10239f5407e0bf43e2056f10cab5386d7319e220478fbed6a8323", + "size": 5792282, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.7.4" + }, + { + "digest": "sha256:3a91ffcf88eadccd4b196138e3c783070af7e5239688128d2c7ed400289574e4", + "size": 26568003, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tPROFILE_TASK='-m test.regrtest --pgo \t\t\ttest_array \t\t\ttest_base64 \t\t\ttest_binascii \t\t\ttest_binhex \t\t\ttest_binop \t\t\ttest_bytes \t\t\ttest_c_locale_coercion \t\t\ttest_class \t\t\ttest_cmath \t\t\ttest_codecs \t\t\ttest_compile \t\t\ttest_complex \t\t\ttest_csv \t\t\ttest_decimal \t\t\ttest_dict \t\t\ttest_float \t\t\ttest_fstring \t\t\ttest_hashlib \t\t\ttest_io \t\t\ttest_iter \t\t\ttest_json \t\t\ttest_long \t\t\ttest_math \t\t\ttest_memoryview \t\t\ttest_pickle \t\t\ttest_re \t\t\ttest_set \t\t\ttest_slice \t\t\ttest_struct \t\t\ttest_threading \t\t\ttest_time \t\t\ttest_traceback \t\t\ttest_unicode \t\t' \t&& make install \t&& ldconfig \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:ee82cc8e15068a2e57dec1805ebc11ae639bf10d967bf1b859794756916bd0de", + "size": 235, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=19.2.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/404c9418e33c5031b1a9ab623168b3e8a2ed8c88/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=56bb63d3cf54e7444351256f72a60f575f6d8c7f1faacffae33167afc8e7609d" + }, + { + "digest": "sha256:f0fe55e0f617741a0138eef3b2d1a300da7a877d9e53206d3c65d7328dec0a59", + "size": 1863155, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:1f7879d098d1a0827ae0c4e9cb37cb9904b08c29bc928204edd19184b551db56", + "size": 8241010, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua /bin/sh -c DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 && pip install requests seesaw warcio && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b20fd304fc72c42021381facad6cd033901da2d61f1ccc3bb54cd8f046ba9a49", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:c40d262015cba4bfa6bf2cc026211d328365f789cfbeeae8492b1f074ce4cbe5", + "size": 494028, + "instruction": "COPY dir:98ce9947813e468c8a9081f5660590d4ac67812bdeb182db1878fb0760965d36 in /grab " + }, + { + "digest": "sha256:dc0a587cd40463ecb81919c449e01cf1a4c170970832eef07a9921a2d2267278", + "size": 139, + "instruction": "/bin/sh -c ln -s /usr/local/bin/wget-lua /grab/wget-lua" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355151353, + "status": "stale", + "last_pulled": "2020-10-09T05:52:28.669085Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.info.json new file mode 100644 index 0000000000..b8df07ca17 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/singstar-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 83570324, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:2631fb2f8a664f4e0a027018479146367f9d101011e6bb926b9c14e3bf8c1bb7", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355151353, + "status": "stale", + "last_pulled": "2020-10-09T05:52:28.669085Z", + "last_pushed": null + } + ], + "last_updated": "2020-01-21T23:35:49.453971Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 8355754, + "full_size": 355151353, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-09T05:52:28.669085Z", + "tag_last_pushed": "2020-01-21T23:35:49.453971Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/info.json new file mode 100644 index 0000000000..19c4ac791a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "snscrape-upstream", + "namespace": "warcforceone", + "repository_type": null, + "status": 0, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 0, + "last_updated": null, + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/sources.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/sources.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/tags.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape-upstream/tags.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/Dockerfile new file mode 100644 index 0000000000..de46bd735c --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3-alpine +COPY . /repo +RUN apk add --no-cache git gcc libxml2-dev musl-dev libxslt-dev g++ re2-dev \ + && ln -s /usr/include/libxml2/libxml /usr/include/libxml \ + && pip3 install /repo \ + && rm -rf /repo +ENV PYTHONUNBUFFERED=1 +WORKDIR /data +ENTRYPOINT ["snscrape"] diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/info.json new file mode 100644 index 0000000000..4ebe054c71 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "snscrape", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 21, + "last_updated": "2020-06-03T01:31:31.271076Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "# snscrape\nsnscrape is a scraper for social networking services (SNS). It scrapes things like user profiles, hashtags, or searches and returns the discovered items, e.g. the relevant posts. \n\nThe following services are currently supported:\n* Facebook: user profiles and groups\n* Gab: user profile posts, media, and comments\n* Google+: user profiles\n* Instagram: user profiles, hashtags, and locations\n* Twitter: user profiles, hashtags, searches, threads, and lists (members as well as posts)\n* VKontakte: user profiles\n\n## Requirements\nsnscrape requires Python 3.6 or higher. The Python package dependencies are installed automatically when you install snscrape.\n\nNote that one of the dependencies, lxml, also requires libxml2 and libxslt to be installed.\n\n## Installation\n pip3 install snscrape\n\nIf you want to use the development version:\n\n pip3 install git+https://github.com/JustAnotherArchivist/snscrape.git\n\n## Usage\nTo get all tweets by Jason Scott (@textfiles):\n\n snscrape twitter-user textfiles\n\nIt's usually useful to redirect the output to a file for further processing, e.g. in bash using the filename `@textfiles-tweets`:\n```bash\nsnscrape twitter-user textfiles >twitter-@textfiles\n```\n\nTo get the latest 100 tweets with the hashtag #archiveteam:\n\n snscrape --max-results 100 twitter-hashtag archiveteam\n\n`snscrape --help` or `snscrape --help` provides details on the available options. `snscrape --help` also lists all available modules.\n\nIt is also possible to use snscrape as a library in Python, but this is currently undocumented.\n\n## Issue reporting\nIf you discover an issue with snscrape, please report it at . If possible please run snscrape with `-vv` and `--dump-locals` and include the log output as well as the dump files referenced in the log in the issue. Note that the files may contain sensitive information in some cases and could potentially be used to identify you (e.g. if the service includes your IP address in its response). If you prefer to arrange a file transfer privately, just mention that in the issue.\n\n## License\nThis program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.\n\nThis program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License along with this program. If not, see .\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/sources.json new file mode 100644 index 0000000000..7a85be11bb --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/4f6a6ab3-a476-4a20-999d-fc7125abc99b/" + ], + "channel": "Stable", + "image": "warcforceone/snscrape", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "snscrape", + "resource_uri": "/api/build/v1/source/014b1e4e-fe85-4488-bed2-230845af53f2/", + "state": "Success", + "uuid": "014b1e4e-fe85-4488-bed2-230845af53f2" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags.json new file mode 100644 index 0000000000..d6a16fbe4e --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 63842285, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:c239ab427c23d0016355efc8be0dbfe2f4125b9e88b3efb266567f97081c3ba2", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 111925699, + "status": "active", + "last_pulled": "2021-02-11T13:42:49.539731Z", + "last_pushed": null + } + ], + "last_updated": "2020-06-03T01:31:05.473186Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7474149, + "full_size": 111925699, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-11T13:42:49.539731Z", + "tag_last_pushed": "2020-06-03T01:31:05.473186Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.images.json new file mode 100644 index 0000000000..64f090f49d --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.images.json @@ -0,0 +1,101 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:c239ab427c23d0016355efc8be0dbfe2f4125b9e88b3efb266567f97081c3ba2", + "layers": [ + { + "digest": "sha256:cbdbe7a5bc2a134ca8ec91be58565ec07d037386d1f1d8385412d224deafca08", + "size": 2813316, + "instruction": "ADD file:b91adb67b670d3a6ff9463e48b7def903ed516be66fc4282d22c53e41512be49 in / " + }, + { + "size": 0, + "instruction": " CMD [\"/bin/sh\"]" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:26ebcd19a4e3221ee54d309346df0869af0d74a25920287f70b0ee1da1bf75de", + "size": 301276, + "instruction": "/bin/sh -c apk add --no-cache ca-certificates" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.3" + }, + { + "digest": "sha256:a29d43ca1bb44ea27c7b0843b49cc454778069dc33caf9d4133f7ecb7df5c75d", + "size": 22282301, + "instruction": "/bin/sh -c set -ex \t&& apk add --no-cache --virtual .fetch-deps \t\tgnupg \t\ttar \t\txz \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& apk add --no-cache --virtual .build-deps \t\tbluez-dev \t\tbzip2-dev \t\tcoreutils \t\tdpkg-dev dpkg \t\texpat-dev \t\tfindutils \t\tgcc \t\tgdbm-dev \t\tlibc-dev \t\tlibffi-dev \t\tlibnsl-dev \t\tlibtirpc-dev \t\tlinux-headers \t\tmake \t\tncurses-dev \t\topenssl-dev \t\tpax-utils \t\treadline-dev \t\tsqlite-dev \t\ttcl-dev \t\ttk \t\ttk-dev \t\tutil-linux-dev \t\txz-dev \t\tzlib-dev \t&& apk del --no-network .fetch-deps \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tEXTRA_CFLAGS=\"-DTHREAD_STACK_SIZE=0x100000\" \t\tLDFLAGS=\"-Wl,--strip-all\" \t&& make install \t\t&& find /usr/local -type f -executable -not \\( -name '*tkinter*' \\) -exec scanelf --needed --nobanner --format '%n#p' '{}' ';' \t\t| tr ',' '\\n' \t\t| sort -u \t\t| awk 'system(\"[ -e /usr/local/lib/\" $1 \" ]\") == 0 { next } { print \"so:\" $1 }' \t\t| xargs -rt apk add --no-cache --virtual .python-rundeps \t&& apk del --no-network .build-deps \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:979dbbcf63e040129349367e745db1b372a14883fc106b1355b934eda6795ed0", + "size": 229, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.1.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/eff16c878c7fd6b688b9b4c4267695cf1a0bf01b/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=b3153ec0cf7b7bbf9556932aa37e4981c35dc2a2c501d70d91d2795aa532be79" + }, + { + "digest": "sha256:30beed04940cb5afa7986dd5ffe0a0a022a56ad57a081f9db889a77d799b9af3", + "size": 1931848, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum -c -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "digest": "sha256:5f5d0c6dec56294ed701dd3de3f81ad2c7667ff4cf035a873480b8b4c8e6502a", + "size": 125813, + "instruction": "COPY dir:1176b046bb3c06fc1da51814ddfdfe26713e0d2a03bb5c69d70f98354ef919ca in /repo " + }, + { + "digest": "sha256:87ef658e315083a810d5313db4cf7200d5b4bce20ce90bff19ff64a5364ffe3b", + "size": 84470825, + "instruction": "/bin/sh -c apk add --no-cache git gcc libxml2-dev musl-dev libxslt-dev g++ re2-dev && ln -s /usr/include/libxml2/libxml /usr/include/libxml && pip3 install /repo && rm -rf /repo" + }, + { + "size": 0, + "instruction": " ENV PYTHONUNBUFFERED=1" + }, + { + "digest": "sha256:3963cf561ac034f0842949c9354cdb7e123af1c1031fe09b5aae5217cc61196b", + "size": 91, + "instruction": "WORKDIR /data" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"snscrape\"]" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 111925699, + "status": "active", + "last_pulled": "2021-02-11T13:42:49.539731Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.info.json new file mode 100644 index 0000000000..af6cfa7cbe --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/snscrape/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 63842285, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:c239ab427c23d0016355efc8be0dbfe2f4125b9e88b3efb266567f97081c3ba2", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 111925699, + "status": "active", + "last_pulled": "2021-02-11T13:42:49.539731Z", + "last_pushed": null + } + ], + "last_updated": "2020-06-03T01:31:05.473186Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 7474149, + "full_size": 111925699, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-11T13:42:49.539731Z", + "tag_last_pushed": "2020-06-03T01:31:05.473186Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/Dockerfile new file mode 100644 index 0000000000..ec760688d1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM warcforceone/grab-base +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua \ + && chmod +x /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/info.json new file mode 100644 index 0000000000..6663377d49 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "soup-io-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 935469, + "last_updated": "2020-07-21T16:00:50.757421Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "soup-io-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [soup.io](http://archiveteam.org/index.php?title=soup.io)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"soup.io\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n pip install --upgrade seesaw zstandard\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/soup-io-grab.git; cd soup-io-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/soup-io-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, soup-io-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/soup-io-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/soup-io-grab\n cd soup-io-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC #nosoupforyou.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/sources.json new file mode 100644 index 0000000000..a11a8ac2fc --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/a864717e-757c-4c51-8416-a559d71141e8/" + ], + "channel": "Stable", + "image": "warcforceone/soup-io-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "soup-io-grab", + "resource_uri": "/api/build/v1/source/4c5f3c58-96b9-4755-93cb-fb1db6fb32b0/", + "state": "Success", + "uuid": "4c5f3c58-96b9-4755-93cb-fb1db6fb32b0" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags.json new file mode 100644 index 0000000000..c30f793134 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 108218312, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:07978a1e9bcf0eaa98f4492259776f002f54a2f331813ece71bd9e3d0b8b82b1", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 375961763, + "status": "stale", + "last_pulled": "2020-10-16T03:32:16.886923Z", + "last_pushed": null + } + ], + "last_updated": "2020-07-21T16:00:50.448121Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9437511, + "full_size": 375961763, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-16T03:32:16.886923Z", + "tag_last_pushed": "2020-07-21T16:00:50.448121Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.images.json new file mode 100644 index 0000000000..47c9569562 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:07978a1e9bcf0eaa98f4492259776f002f54a2f331813ece71bd9e3d0b8b82b1", + "layers": [ + { + "digest": "sha256:5ae19949497e04289972756fe51cfac1a72b04fe2709e85a615945035c5a9a61", + "size": 50380042, + "instruction": "ADD file:2cddee716e84c40540a69c48051bd2dcf6cd3bd02a3e399334e97f20a77126ff in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ed3d96a2798e8837be24597cabf44ce25585cb9db1d749299cb06d51349ea5c2", + "size": 7804488, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f1213685078145f6136360475dbaffd0f86dfe92133a7bc26d79602980b255dd", + "size": 9978163, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:1a9ad5d5550bdff7db4c3d035bf9550bcd1de06a7f178a26de1d082591a5b956", + "size": 51765388, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:6f18049a0455d5e717b580354b515d9ac661e1a28d6ca9d6f7bc85c0dd17a7cf", + "size": 192264327, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibgeoip-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:ce39fa9d79d10239f5407e0bf43e2056f10cab5386d7319e220478fbed6a8323", + "size": 5792282, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.7.4" + }, + { + "digest": "sha256:3a91ffcf88eadccd4b196138e3c783070af7e5239688128d2c7ed400289574e4", + "size": 26568003, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tPROFILE_TASK='-m test.regrtest --pgo \t\t\ttest_array \t\t\ttest_base64 \t\t\ttest_binascii \t\t\ttest_binhex \t\t\ttest_binop \t\t\ttest_bytes \t\t\ttest_c_locale_coercion \t\t\ttest_class \t\t\ttest_cmath \t\t\ttest_codecs \t\t\ttest_compile \t\t\ttest_complex \t\t\ttest_csv \t\t\ttest_decimal \t\t\ttest_dict \t\t\ttest_float \t\t\ttest_fstring \t\t\ttest_hashlib \t\t\ttest_io \t\t\ttest_iter \t\t\ttest_json \t\t\ttest_long \t\t\ttest_math \t\t\ttest_memoryview \t\t\ttest_pickle \t\t\ttest_re \t\t\ttest_set \t\t\ttest_slice \t\t\ttest_struct \t\t\ttest_threading \t\t\ttest_time \t\t\ttest_traceback \t\t\ttest_unicode \t\t' \t&& make install \t&& ldconfig \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:ee82cc8e15068a2e57dec1805ebc11ae639bf10d967bf1b859794756916bd0de", + "size": 235, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=19.2.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/404c9418e33c5031b1a9ab623168b3e8a2ed8c88/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=56bb63d3cf54e7444351256f72a60f575f6d8c7f1faacffae33167afc8e7609d" + }, + { + "digest": "sha256:f0fe55e0f617741a0138eef3b2d1a300da7a877d9e53206d3c65d7328dec0a59", + "size": 1863155, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:1f7879d098d1a0827ae0c4e9cb37cb9904b08c29bc928204edd19184b551db56", + "size": 8241010, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua /bin/sh -c DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 && pip install requests seesaw warcio && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b20fd304fc72c42021381facad6cd033901da2d61f1ccc3bb54cd8f046ba9a49", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:382c0137254d4fd2f81e75afefea06694fb89311e992737472f84755636d19ed", + "size": 20479225, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:5e918eb0fe055c19cc99620668ff45842d17b9d305c68366f63f26e3283acb13", + "size": 41652, + "instruction": "COPY dir:b147f69f20381c453cb34683cd66c011613808348e4d9795b15838e7b0ccf866 in /grab " + }, + { + "digest": "sha256:c92c648cb36c395ad8a21fa46ca48d83d6936e8122e4f2a5587df3e4a383c1be", + "size": 783700, + "instruction": "/bin/sh -c wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua && chmod +x /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 375961763, + "status": "stale", + "last_pulled": "2020-10-16T03:32:16.886923Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.info.json new file mode 100644 index 0000000000..32a623d08a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/soup-io-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 108218312, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:07978a1e9bcf0eaa98f4492259776f002f54a2f331813ece71bd9e3d0b8b82b1", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 375961763, + "status": "stale", + "last_pulled": "2020-10-16T03:32:16.886923Z", + "last_pushed": null + } + ], + "last_updated": "2020-07-21T16:00:50.448121Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9437511, + "full_size": 375961763, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-16T03:32:16.886923Z", + "tag_last_pushed": "2020-07-21T16:00:50.448121Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/Dockerfile new file mode 100644 index 0000000000..b2880e3030 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/Dockerfile @@ -0,0 +1,4 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at +RUN sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python3.8/site-packages/seesaw/tracker.py diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/info.json new file mode 100644 index 0000000000..fbcd7a815a --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "tencent-weibo-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 94271777, + "last_updated": "2020-10-31T00:33:22.019703Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "tencent-weibo-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [\u817e\u8baf\u5fae\u535a](http://archiveteam.org/index.php?title=\u817e\u8baf\u5fae\u535a)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"\u817e\u8baf\u5fae\u535a\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/tencent-weibo-grab.git; cd tencent-weibo-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/tencent-weibo-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, tencent-weibo-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/tencent-weibo-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/tencent-weibo-grab\n cd tencent-weibo-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at IRC hackint #twocents.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/sources.json new file mode 100644 index 0000000000..dc7aa42966 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/b376fd6b-3c6d-44da-8a44-9d378adf729f/" + ], + "channel": "Stable", + "image": "warcforceone/tencent-weibo-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "tencent-weibo-grab", + "resource_uri": "/api/build/v1/source/bf630818-4cbb-4c63-89be-ee056a848448/", + "state": "Failed", + "uuid": "bf630818-4cbb-4c63-89be-ee056a848448" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags.json new file mode 100644 index 0000000000..845fa9771d --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 117793833, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b276f6f881d8b119906c3bed970b5fed871f323e903b26d39c4201f86acb8aec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353998513, + "status": "active", + "last_pulled": "2021-03-03T00:02:23.186923Z", + "last_pushed": "2020-10-31T00:33:13.098519Z" + } + ], + "last_updated": "2020-10-31T00:33:13.098519Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9852521, + "full_size": 353998513, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T00:02:23.186923Z", + "tag_last_pushed": "2020-10-31T00:33:13.098519Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.images.json new file mode 100644 index 0000000000..80eb1440e0 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:b276f6f881d8b119906c3bed970b5fed871f323e903b26d39c4201f86acb8aec", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:7f7c8879a275c860b98da629884e116f34e7531db5595b6d8c22653883506ed2", + "size": 2120572, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:b6c4e663f7617a6ea954dca6864617839abd937eba3eddcc32a145ceb34bd492", + "size": 15176496, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:9428bde242c46afca174985448a1fbdc1da4d1e75b2294486e478f8f56154b83", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:2f95c1eb578f88241406823b2fe4e90170d3adc2c6669cd6d6cf6aa5e6c4caa0", + "size": 66803, + "instruction": "COPY dir:d4b7e6a37f8f01dd3a5174f6dd485f0f802f21159079b701ad7effa263238ee0 in /grab " + }, + { + "digest": "sha256:2b97698c20375c740fbc41903f033f6e1fa764121b7faef338764b0d612ce794", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + }, + { + "digest": "sha256:f5aaa355ef7d77124c27b1017fb29a75a886262b86409e211270cab95aaa3bfb", + "size": 2895, + "instruction": "/bin/sh -c sed -i 's|DEFAULT_RETRY_DELAY = 60$|DEFAULT_RETRY_DELAY = 0|;s|self\\.retry_delay += 10|self.retry_delay += 0.1|' /usr/local/lib/python3.8/site-packages/seesaw/tracker.py" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 353998513, + "status": "active", + "last_pulled": "2021-03-03T00:02:23.186923Z", + "last_pushed": "2020-10-31T00:33:13.098519Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.info.json new file mode 100644 index 0000000000..103d6c1daa --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/tencent-weibo-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 117793833, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:b276f6f881d8b119906c3bed970b5fed871f323e903b26d39c4201f86acb8aec", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 353998513, + "status": "active", + "last_pulled": "2021-03-03T00:02:23.186923Z", + "last_pushed": "2020-10-31T00:33:13.098519Z" + } + ], + "last_updated": "2020-10-31T00:33:13.098519Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9852521, + "full_size": 353998513, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-03-03T00:02:23.186923Z", + "tag_last_pushed": "2020-10-31T00:33:13.098519Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/Dockerfile new file mode 100644 index 0000000000..ec760688d1 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/Dockerfile @@ -0,0 +1,9 @@ +FROM warcforceone/grab-base +RUN echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 \ + && pip install zstandard +COPY . /grab +RUN wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua \ + && chmod +x /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/info.json new file mode 100644 index 0000000000..c8c05f5068 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "theartistunion-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 139872, + "last_updated": "2020-10-15T14:40:26.351961Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "theartistunion-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [The Artist Union](http://archiveteam.org/index.php?title=The_Artist_Union)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"The Artist Union\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/theartistunion-grab.git; cd theartistunion-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/theartistunion-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, theartistunion-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/theartistunion-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/theartistunion-grab\n cd theartistunion-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #archiveteam.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/sources.json new file mode 100644 index 0000000000..9c92239cdf --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/d969e438-189e-4462-a681-b5fb487e80e7/" + ], + "channel": "Stable", + "image": "warcforceone/theartistunion-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "theartistunion-grab", + "resource_uri": "/api/build/v1/source/f55a42ba-2bed-4dc0-b98c-5434e2bd9dc6/", + "state": "Failed", + "uuid": "f55a42ba-2bed-4dc0-b98c-5434e2bd9dc6" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags.json new file mode 100644 index 0000000000..76a932ca67 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 111290282, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:6184157bd69442553c4492f8eaa324d3ae36d279fae860cd877defd93d7c0f03", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369356100, + "status": "active", + "last_pulled": "2021-02-20T02:22:08.491635Z", + "last_pushed": "2020-10-15T14:40:25.796092Z" + } + ], + "last_updated": "2020-10-15T14:40:25.796092Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9555565, + "full_size": 369356100, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:08.491635Z", + "tag_last_pushed": "2020-10-15T14:40:25.796092Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.images.json new file mode 100644 index 0000000000..5dc5f2bfbc --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.images.json @@ -0,0 +1,143 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:6184157bd69442553c4492f8eaa324d3ae36d279fae860cd877defd93d7c0f03", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.8.6" + }, + { + "digest": "sha256:b453803265bc2d6b00650681679288d8871fdaba67aceb28881c5dcb1fbaf816", + "size": 18188917, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\t-o \\( -type f -a -name 'wininst-*.exe' \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:a49354af7e01edfaa97954d2261cfc4d388884d0796f2f7332c44ee8d78f0d28", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.3" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/fa7dc83944936bf09a0e4cb5d5ec852c0d256599/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=6e0bb0a2c2533361d7f297ed547237caf1b7507f197835974c0dd7eba998c53c" + }, + { + "digest": "sha256:3eb2f327eb62905a17b993e9807eeb1c85f9c0172b0721b089586c9fdd4acaee", + "size": 2119480, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:0a47d90c3940a5dd80561f5659e27a5a7f8f7675d95420319ca74039ed94a671", + "size": 15164957, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/bvj4ouZsnftrFk-oRZp0hfP-3Tan5H-z/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b538559a01f82dea3b4dfad9ae58884da104ac09290b66f24874efe1a1dd458b", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:78145b7a069bdc3c517a0c6d48ab2d29f6110a1fa7235aab8f7bc597d30c70a2", + "size": 14111453, + "instruction": "/bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install zstandard" + }, + { + "digest": "sha256:52f9e16adb32d36567a9bd8113b6ebe970f4d6e4b9e8d491dda607e315923e7b", + "size": 544911, + "instruction": "COPY dir:4eeba284110b968df3c1a9b19ac0d92d269c9bf5f574a47c800feb0ae7d60d19 in /grab " + }, + { + "digest": "sha256:a50420f2f9cd1dee61d17b1e20d27a30536fe4f024be74469f0b9588fe67876a", + "size": 783695, + "instruction": "/bin/sh -c wget -O /grab/wget-at http://xor.meo.ws/-qt0VqH8KqsCm5xAkw1Pc7oeXToeyU0u/wget-lua && chmod +x /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 369356100, + "status": "active", + "last_pulled": "2021-02-20T02:22:08.491635Z", + "last_pushed": "2020-10-15T14:40:25.796092Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.info.json new file mode 100644 index 0000000000..191bb71b92 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/theartistunion-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 111290282, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:6184157bd69442553c4492f8eaa324d3ae36d279fae860cd877defd93d7c0f03", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 369356100, + "status": "active", + "last_pulled": "2021-02-20T02:22:08.491635Z", + "last_pushed": "2020-10-15T14:40:25.796092Z" + } + ], + "last_updated": "2020-10-15T14:40:25.796092Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9555565, + "full_size": 369356100, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-20T02:22:08.491635Z", + "tag_last_pushed": "2020-10-15T14:40:25.796092Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/Dockerfile new file mode 100644 index 0000000000..892855d054 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/Dockerfile @@ -0,0 +1,5 @@ +FROM warcforceone/grab-base +COPY . /grab +RUN ln -s /usr/local/bin/wget-lua /grab/wget-lua +# RUN wget -O /grab/wget-lua http://xor.meo.ws/vUO6LyuhBlMOqGUjZ3sFQCqUcR83pl9N/wget-lua \ +# && chmod +x /grab/wget-lua diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/info.json new file mode 100644 index 0000000000..21d7a55032 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "yahoogroups-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 251333, + "last_updated": "2019-12-12T17:52:18.779336Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "yahoogroups-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [Yahoo! Groups](http://archiveteam.org/index.php?title=Yahoo! Groups)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"Yahoo! Groups\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n pip install --upgrade seesaw\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n pip install --upgrade seesaw\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/yahoogroups-grab.git; cd yahoogroups-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/yahoogroups-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n apt-get update && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n\nAnd you may also need to update your Python's warcio if you find screen detaching immediately after running the job in Ubuntu 18.04 Bionic and above:\n\n pip install warcio --upgrade\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will the EPEL repository to be enabled.\n\n yum -y install autoconf automake flex gnutls-devel lua-devel python-pip zlib-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, yahoogroups-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/yahoogroups-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/yahoogroups-grab\n cd yahoogroups-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at irc.efnet.org #yahoosucks.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/sources.json new file mode 100644 index 0000000000..7838f6aafd --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/ad9616d0-44c5-47e1-88a7-0a22a86b6c6f/" + ], + "channel": "Stable", + "image": "warcforceone/yahoogroups-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "yahoogroups-grab", + "resource_uri": "/api/build/v1/source/216b5af9-efed-41f6-832b-60c2990c68df/", + "state": "Success", + "uuid": "216b5af9-efed-41f6-832b-60c2990c68df" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags.json new file mode 100644 index 0000000000..1bbe1f7f57 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 76930544, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:90a0671b8c71ac407a8899788ff79aa9b8ab3d37c583e1dc7b48e8ae9325ce41", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355368779, + "status": "stale", + "last_pulled": "2020-10-31T02:42:16.467439Z", + "last_pushed": null + } + ], + "last_updated": "2019-12-12T17:52:18.60529Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 8049288, + "full_size": 355368779, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-31T02:42:16.467439Z", + "tag_last_pushed": "2019-12-12T17:52:18.60529Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.images.json new file mode 100644 index 0000000000..339c8f5ac0 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.images.json @@ -0,0 +1,138 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:90a0671b8c71ac407a8899788ff79aa9b8ab3d37c583e1dc7b48e8ae9325ce41", + "layers": [ + { + "digest": "sha256:5ae19949497e04289972756fe51cfac1a72b04fe2709e85a615945035c5a9a61", + "size": 50380042, + "instruction": "ADD file:2cddee716e84c40540a69c48051bd2dcf6cd3bd02a3e399334e97f20a77126ff in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:ed3d96a2798e8837be24597cabf44ce25585cb9db1d749299cb06d51349ea5c2", + "size": 7804488, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:f1213685078145f6136360475dbaffd0f86dfe92133a7bc26d79602980b255dd", + "size": 9978163, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:1a9ad5d5550bdff7db4c3d035bf9550bcd1de06a7f178a26de1d082591a5b956", + "size": 51765388, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:6f18049a0455d5e717b580354b515d9ac661e1a28d6ca9d6f7bc85c0dd17a7cf", + "size": 192264327, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibgeoip-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:ce39fa9d79d10239f5407e0bf43e2056f10cab5386d7319e220478fbed6a8323", + "size": 5792282, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.7.4" + }, + { + "digest": "sha256:3a91ffcf88eadccd4b196138e3c783070af7e5239688128d2c7ed400289574e4", + "size": 26568003, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t\tPROFILE_TASK='-m test.regrtest --pgo \t\t\ttest_array \t\t\ttest_base64 \t\t\ttest_binascii \t\t\ttest_binhex \t\t\ttest_binop \t\t\ttest_bytes \t\t\ttest_c_locale_coercion \t\t\ttest_class \t\t\ttest_cmath \t\t\ttest_codecs \t\t\ttest_compile \t\t\ttest_complex \t\t\ttest_csv \t\t\ttest_decimal \t\t\ttest_dict \t\t\ttest_float \t\t\ttest_fstring \t\t\ttest_hashlib \t\t\ttest_io \t\t\ttest_iter \t\t\ttest_json \t\t\ttest_long \t\t\ttest_math \t\t\ttest_memoryview \t\t\ttest_pickle \t\t\ttest_re \t\t\ttest_set \t\t\ttest_slice \t\t\ttest_struct \t\t\ttest_threading \t\t\ttest_time \t\t\ttest_traceback \t\t\ttest_unicode \t\t' \t&& make install \t&& ldconfig \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' + \t&& rm -rf /usr/src/python \t\t&& python3 --version" + }, + { + "digest": "sha256:ee82cc8e15068a2e57dec1805ebc11ae639bf10d967bf1b859794756916bd0de", + "size": 235, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=19.2.1" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/404c9418e33c5031b1a9ab623168b3e8a2ed8c88/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=56bb63d3cf54e7444351256f72a60f575f6d8c7f1faacffae33167afc8e7609d" + }, + { + "digest": "sha256:f0fe55e0f617741a0138eef3b2d1a300da7a877d9e53206d3c65d7328dec0a59", + "size": 1863155, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:1f7879d098d1a0827ae0c4e9cb37cb9904b08c29bc928204edd19184b551db56", + "size": 8241010, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/zEMhOKrjwKi69SIWBdxiQGJ2IpzzSIx4/wget-lua /bin/sh -c DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 && pip install requests seesaw warcio && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:b20fd304fc72c42021381facad6cd033901da2d61f1ccc3bb54cd8f046ba9a49", + "size": 93, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "digest": "sha256:38b24b2a9d9cee77c70f0137ceda8db9272664d448be2bf9ac228365115cb310", + "size": 711454, + "instruction": "COPY dir:83b97096fc1102fbf1d1f73d58e0542aeb0f4c20c0b93272dd93132022f0efa5 in /grab " + }, + { + "digest": "sha256:21055953d9a4a5e44ba85d2f0e1664e426d9faf2698e5f75e3a9dca9cac00d65", + "size": 139, + "instruction": "/bin/sh -c ln -s /usr/local/bin/wget-lua /grab/wget-lua" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 355368779, + "status": "stale", + "last_pulled": "2020-10-31T02:42:16.467439Z", + "last_pushed": null + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.info.json new file mode 100644 index 0000000000..5eb94fde5f --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/yahoogroups-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 76930544, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:90a0671b8c71ac407a8899788ff79aa9b8ab3d37c583e1dc7b48e8ae9325ce41", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 355368779, + "status": "stale", + "last_pulled": "2020-10-31T02:42:16.467439Z", + "last_pushed": null + } + ], + "last_updated": "2019-12-12T17:52:18.60529Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 8049288, + "full_size": 355368779, + "v2": true, + "tag_status": "stale", + "tag_last_pulled": "2020-10-31T02:42:16.467439Z", + "tag_last_pushed": "2019-12-12T17:52:18.60529Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/Dockerfile b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/Dockerfile new file mode 100644 index 0000000000..551df685a7 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/Dockerfile @@ -0,0 +1,3 @@ +FROM atdr.meo.ws/archiveteam/grab-base-df +COPY . /grab +RUN ln -fs /usr/local/bin/wget-lua /grab/wget-at diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/info.json new file mode 100644 index 0000000000..7d0df1abd3 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/info.json @@ -0,0 +1,25 @@ +{ + "user": "warcforceone", + "name": "youtube-playlistnotes-grab", + "namespace": "warcforceone", + "repository_type": "image", + "status": 1, + "description": "", + "is_private": false, + "is_automated": false, + "can_edit": false, + "star_count": 0, + "pull_count": 11642978, + "last_updated": "2020-10-31T11:04:04.603216Z", + "is_migrated": false, + "collaborator_count": 0, + "affiliation": null, + "hub_user": "warcforceone", + "has_starred": false, + "full_description": "youtube-playlistnotes-grab\n=============\n\nMore information about the archiving project can be found on the ArchiveTeam wiki: [YouTube](http://archiveteam.org/index.php?title=YouTube)\n\nSetup instructions\n=========================\n\nBe sure to replace `YOURNICKHERE` with the nickname that you want to be shown as, on the tracker. You don't need to register it, just pick a nickname you like.\n\nIn most of the below cases, there will be a web interface running at http://localhost:8001/. If you don't know or care what this is, you can just ignore it\u2014otherwise, it gives you a fancy view of what's going on.\n\n**If anything goes wrong while running the commands below, please scroll down to the bottom of this page. There's troubleshooting information there.**\n\nRunning with a warrior\n-------------------------\n\nFollow the [instructions on the ArchiveTeam wiki](http://archiveteam.org/index.php?title=Warrior) for installing the Warrior, and select the \"YouTube Playlist Notes\" project in the Warrior interface.\n\nRunning without a warrior\n-------------------------\nTo run this outside the warrior, clone this repository, cd into its directory and run:\n\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n ./get-wget-lua.sh\n\nthen start downloading with:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nFor more options, run:\n\n run-pipeline3 --help\n\nIf you don't have root access and/or your version of pip is very old, you can replace \"pip install --upgrade seesaw\" with:\n\n wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw\n\nso that pip and seesaw are installed in your home, then run\n\n ~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE\n\nRunning multiple instances on different IPs\n-------------------------------------------\n\nThis feature requires seesaw version 0.0.16 or greater. Use `pip install --upgrade seesaw` to upgrade.\n\nUse the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace the IP address with your own).\n\nExample of running 2 threads, no web interface, and Wget binding of IP address:\n\n run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6\n\nDistribution-specific setup\n-------------------------\n### For Debian/Ubuntu:\n\nPackage `libzstd-dev` version 1.4.4 is required which is currently available from `buster-backports`.\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n python3 -m pip install setuptools wheel\n python3 -m pip install --upgrade seesaw zstandard requests\n su -c \"cd /home/archiveteam; git clone https://github.com/ArchiveTeam/youtube-playlistnotes-grab.git; cd youtube-playlistnotes-grab; ./get-wget-lua.sh\" archiveteam\n screen su -c \"cd /home/archiveteam/youtube-playlistnotes-grab/; run-pipeline3 pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam\n [... ctrl+A D to detach ...]\n\nIn __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package was renamed to `libgnutls28-dev`. So, you need to do the following instead:\n\n adduser --system --group --shell /bin/bash archiveteam\n echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list\n apt-get update \\\n && apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \\\n && apt-get -t buster-backports install zstd libzstd-dev libzstd1\n [... pretty much the same as above ...]\n\nWget-lua is also available on [ArchiveTeam's PPA](https://launchpad.net/~archiveteam/+archive/wget-lua) for Ubuntu.\n\n### For CentOS:\n\nEnsure that you have the CentOS equivalent of bzip2 installed as well. You will need the EPEL repository to be enabled.\n\n yum -y groupinstall \"Development Tools\"\n yum -y install gnutls-devel lua-devel python-pip zlib-devel zstd libzstd-devel git-core gperf lua-socket luarocks texinfo git rsync gettext-devel\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\nTested with EL7 repositories.\n\n### For Fedora:\n\nThe same as CentOS but with \"dnf\" instead of \"yum\". Did not successfully test compiling, so far.\n\n### For openSUSE:\n\n zypper install liblua5_1 lua51 lua51-devel screen python-pip libgnutls-devel bzip2 python-devel gcc make\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n### For OS X:\n\nYou need Homebrew. Ensure that you have the OS X equivalent of bzip2 installed as well.\n\n brew install python lua gnutls\n pip install --upgrade seesaw\n [... pretty much the same as above ...]\n\n**There is a known issue with some packaged versions of rsync. If you get errors during the upload stage, youtube-playlistnotes-grab will not work with your rsync version.**\n\nThis supposedly fixes it:\n\n alias rsync=/usr/local/bin/rsync\n\n### For Arch Linux:\n\nEnsure that you have the Arch equivalent of bzip2 installed as well.\n\n1. Make sure you have `python2-pip` installed.\n2. Install [the wget-lua package from the AUR](https://aur.archlinux.org/packages/wget-lua/). \n3. Run `pip2 install --upgrade seesaw`.\n4. Modify the run-pipeline script in seesaw to point at `#!/usr/bin/python2` instead of `#!/usr/bin/python`.\n5. `useradd --system --group users --shell /bin/bash --create-home archiveteam`\n6. `screen su -c \"cd /home/archiveteam/youtube-playlistnotes-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\" archiveteam`\n\n### For Alpine Linux:\n\n apk add lua5.1 git python bzip2 bash rsync gcc libc-dev lua5.1-dev zlib-dev gnutls-dev autoconf flex make\n python -m ensurepip\n pip install -U seesaw\n git clone https://github.com/ArchiveTeam/youtube-playlistnotes-grab\n cd youtube-playlistnotes-grab; ./get-wget-lua.sh\n run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE\n\n### For FreeBSD:\n\nHonestly, I have no idea. `./get-wget-lua.sh` supposedly doesn't work due to differences in the `tar` that ships with FreeBSD. Another problem is the apparent absence of Lua 5.1 development headers. If you figure this out, please do let us know on IRC (irc.efnet.org #archiveteam).\n\nTroubleshooting\n=========================\n\nBroken? These are some of the possible solutions:\n\n### wget-lua was not successfully built\n\nIf you get errors about `wget.pod` or something similar, the documentation failed to compile - wget-lua, however, compiled fine. Try this:\n\n cd get-wget-lua.tmp\n mv src/wget ../wget-lua\n cd ..\n\nThe `get-wget-lua.tmp` name may be inaccurate. If you have a folder with a similar but different name, use that instead and please let us know on IRC what folder name you had!\n\nOptionally, if you know what you're doing, you may want to use wgetpod.patch.\n\n### Problem with gnutls or openssl during get-wget-lua\n\nPlease ensure that gnutls-dev(el) and openssl-dev(el) are installed.\n\n### ImportError: No module named seesaw\n\nIf you're sure that you followed the steps to install `seesaw`, permissions on your module directory may be set incorrectly. Try the following:\n\n chmod o+rX -R /usr/local/lib/python2.7/dist-packages\n\n### run-pipeline: command not found\n\nInstall `seesaw` using `pip2` instead of `pip`.\n\n pip2 install seesaw\n\n### Issues in the code\n\nIf you notice a bug and want to file a bug report, please use the GitHub issues tracker.\n\nAre you a developer? Help write code for us! Look at our [developer documentation](http://archiveteam.org/index.php?title=Dev) for details.\n\n### Other problems\n\nHave an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC #down-the-tube.\n\n", + "permissions": { + "read": true, + "write": false, + "admin": false + } +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/sources.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/sources.json new file mode 100644 index 0000000000..4f1b81964b --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/sources.json @@ -0,0 +1,18 @@ +[ + { + "autotests": "OFF", + "build_in_farm": true, + "build_settings": [ + "/api/build/v1/setting/4aed3782-e983-4a07-9fdf-040a1ff5da57/" + ], + "channel": "Stable", + "image": "warcforceone/youtube-playlistnotes-grab", + "owner": "ArchiveTeam", + "provider": "Github", + "repo_links": true, + "repository": "youtube-playlistnotes-grab", + "resource_uri": "/api/build/v1/source/e8de687f-0927-4677-a203-abeb641f1442/", + "state": "Success", + "uuid": "e8de687f-0927-4677-a203-abeb641f1442" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags.json new file mode 100644 index 0000000000..9d7153dc02 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags.json @@ -0,0 +1,32 @@ +[ + { + "creator": 6911312, + "id": 119983947, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:9893037e4182af8654f811f2802888178985cc21dcf2e5189a8e7672fbbda3c3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356557380, + "status": "active", + "last_pulled": "2021-02-22T11:15:51.896128Z", + "last_pushed": "2020-10-31T11:04:01.777068Z" + } + ], + "last_updated": "2020-10-31T11:04:01.777068Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9961945, + "full_size": 356557380, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-22T11:15:51.896128Z", + "tag_last_pushed": "2020-10-31T11:04:01.777068Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.images.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.images.json new file mode 100644 index 0000000000..9d3112ed55 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.images.json @@ -0,0 +1,154 @@ +[ + { + "architecture": "amd64", + "features": null, + "variant": null, + "digest": "sha256:9893037e4182af8654f811f2802888178985cc21dcf2e5189a8e7672fbbda3c3", + "layers": [ + { + "digest": "sha256:e4c3d3e4f7b024979a1c12daa4073f6353b2ba92d96418bc90451994927c9bff", + "size": 50395978, + "instruction": "ADD file:6627ad39ea0cb9fcb212342326d14efaff51aece1fd0dc16d5bbcaa25d858622 in / " + }, + { + "size": 0, + "instruction": " CMD [\"bash\"]" + }, + { + "digest": "sha256:101c41d0463bc77661fb3343235b16d536a92d2efb687046164d413e51bd4fc4", + "size": 7811737, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tca-certificates \t\tcurl \t\tnetbase \t\twget \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:8275efcd805f9905d7def23603618236284b0be6b9e47455c638fbfb03fa9208", + "size": 9996326, + "instruction": "/bin/sh -c set -ex; \tif ! command -v gpg > /dev/null; then \t\tapt-get update; \t\tapt-get install -y --no-install-recommends \t\t\tgnupg \t\t\tdirmngr \t\t; \t\trm -rf /var/lib/apt/lists/*; \tfi" + }, + { + "digest": "sha256:751620502a7a2905067c2f32d4982fb9b310b9808670ce82c0e2b40f5307a3ee", + "size": 51829492, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tgit \t\tmercurial \t\topenssh-client \t\tsubversion \t\t\t\tprocps \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:0a5e725150a20c71bdc806daa04c9c9438aed5f5a4dd376f3be66fae213b1d43", + "size": 192263478, + "instruction": "/bin/sh -c set -ex; \tapt-get update; \tapt-get install -y --no-install-recommends \t\tautoconf \t\tautomake \t\tbzip2 \t\tdpkg-dev \t\tfile \t\tg++ \t\tgcc \t\timagemagick \t\tlibbz2-dev \t\tlibc6-dev \t\tlibcurl4-openssl-dev \t\tlibdb-dev \t\tlibevent-dev \t\tlibffi-dev \t\tlibgdbm-dev \t\tlibglib2.0-dev \t\tlibgmp-dev \t\tlibjpeg-dev \t\tlibkrb5-dev \t\tliblzma-dev \t\tlibmagickcore-dev \t\tlibmagickwand-dev \t\tlibmaxminddb-dev \t\tlibncurses5-dev \t\tlibncursesw5-dev \t\tlibpng-dev \t\tlibpq-dev \t\tlibreadline-dev \t\tlibsqlite3-dev \t\tlibssl-dev \t\tlibtool \t\tlibwebp-dev \t\tlibxml2-dev \t\tlibxslt-dev \t\tlibyaml-dev \t\tmake \t\tpatch \t\tunzip \t\txz-utils \t\tzlib1g-dev \t\t\t\t$( \t\t\tif apt-cache show 'default-libmysqlclient-dev' 2>/dev/null | grep -q '^Version:'; then \t\t\t\techo 'default-libmysqlclient-dev'; \t\t\telse \t\t\t\techo 'libmysqlclient-dev'; \t\t\tfi \t\t) \t; \trm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "size": 0, + "instruction": " ENV LANG=C.UTF-8" + }, + { + "digest": "sha256:397dba5694dbe6da34f1abfc75362ca3f23a0421505dd3502885e9bc9a167f42", + "size": 6145350, + "instruction": "/bin/sh -c apt-get update && apt-get install -y --no-install-recommends \t\tlibbluetooth-dev \t\ttk-dev \t\tuuid-dev \t&& rm -rf /var/lib/apt/lists/*" + }, + { + "size": 0, + "instruction": " ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568" + }, + { + "size": 0, + "instruction": " ENV PYTHON_VERSION=3.9.0" + }, + { + "digest": "sha256:b1d09d0eabcb6f4e10ef0d360714bc5c780ea017bbcdc1a7e1a4a617ff8fb431", + "size": 20579241, + "instruction": "/bin/sh -c set -ex \t\t&& wget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\" \t&& wget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\" \t&& export GNUPGHOME=\"$(mktemp -d)\" \t&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys \"$GPG_KEY\" \t&& gpg --batch --verify python.tar.xz.asc python.tar.xz \t&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \t&& rm -rf \"$GNUPGHOME\" python.tar.xz.asc \t&& mkdir -p /usr/src/python \t&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \t&& rm python.tar.xz \t\t&& cd /usr/src/python \t&& gnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\" \t&& ./configure \t\t--build=\"$gnuArch\" \t\t--enable-loadable-sqlite-extensions \t\t--enable-optimizations \t\t--enable-option-checking=fatal \t\t--enable-shared \t\t--with-system-expat \t\t--with-system-ffi \t\t--without-ensurepip \t&& make -j \"$(nproc)\" \t&& make install \t&& rm -rf /usr/src/python \t\t&& find /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \\) \\) \t\t\\) -exec rm -rf '{}' + \t\t&& ldconfig \t\t&& python3 --version" + }, + { + "digest": "sha256:475299e7c7f3a160843a7c349c788b8c9ec2ce9dd3b924e42830a38995955c9d", + "size": 233, + "instruction": "/bin/sh -c cd /usr/local/bin \t&& ln -s idle3 idle \t&& ln -s pydoc3 pydoc \t&& ln -s python3 python \t&& ln -s python3-config python-config" + }, + { + "size": 0, + "instruction": " ENV PYTHON_PIP_VERSION=20.2.4" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/8283828b8fd6f1783daf55a765384e6d8d2c5014/get-pip.py" + }, + { + "size": 0, + "instruction": " ENV PYTHON_GET_PIP_SHA256=2250ab0a7e70f6fd22b955493f7f5cf1ea53e70b584a84a32573644a045b4bfb" + }, + { + "digest": "sha256:d2fe14d8e6bcb0ddb14c3ae2bc50b0f9cad6f42b33935609d160555f3525d71e", + "size": 2120746, + "instruction": "/bin/sh -c set -ex; \t\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum --check --strict -; \t\tpython get-pip.py \t\t--disable-pip-version-check \t\t--no-cache-dir \t\t\"pip==$PYTHON_PIP_VERSION\" \t; \tpip --version; \t\tfind /usr/local -depth \t\t\\( \t\t\t\\( -type d -a \\( -name test -o -name tests -o -name idle_test \\) \\) \t\t\t-o \t\t\t\\( -type f -a \\( -name '*.pyc' -o -name '*.pyo' \\) \\) \t\t\\) -exec rm -rf '{}' +; \trm -f get-pip.py" + }, + { + "size": 0, + "instruction": " CMD [\"python3\"]" + }, + { + "size": 0, + "instruction": " ARG VERSION=20190803.01" + }, + { + "size": 0, + "instruction": " ARG wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua" + }, + { + "size": 0, + "instruction": " ENV LC_ALL=C" + }, + { + "digest": "sha256:76cccca5d2434bc713235d52a22280085ecc46e033071abae779166dd01288d7", + "size": 15354788, + "instruction": "|2 VERSION=20190803.01 wget_lua=http://xor.meo.ws/1hh92cDPD-Rfbd7dUT_AGmsHutAr_QeT/wget-lua /bin/sh -c echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install rsync liblua5.1-0 libluajit-5.1-2 libidn11 lua-socket && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd libzstd-dev libzstd1 && pip install requests seesaw zstandard && wget \"${wget_lua}\" -O /usr/local/bin/wget-lua && chmod +x /usr/local/bin/wget-lua && rm -rf /var/lib/apt/lists/*" + }, + { + "digest": "sha256:23ae9b84d8c637d7be11896ab80072718fff61a7f7779bafdab94a2e1a944695", + "size": 125, + "instruction": "WORKDIR /grab" + }, + { + "size": 0, + "instruction": " STOPSIGNAL SIGINT" + }, + { + "size": 0, + "instruction": " ENTRYPOINT [\"run-pipeline3\" \"--disable-web-server\" \"pipeline.py\"]" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.build-date=2020-10-31T10:57:28Z" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.schema-version=1.0" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-ref=0865fe5498c6ef4730c784dd9c6a4f4e1f98a4f3" + }, + { + "size": 0, + "instruction": " LABEL org.label-schema.vcs-url=https://github.com/ArchiveTeam/grab-base-df.git" + }, + { + "digest": "sha256:445ee09bf57b640913485a6d84d0ed2a65c5952adb92ecb51d195b9d3bb6c824", + "size": 59743, + "instruction": "COPY dir:7f5f04eeaa8256a4187102a73550e799899740cca02802ec30f8045a31d979cf in /grab " + }, + { + "digest": "sha256:8cc11b70302c8774d19f86a1562f21c90094604b8125b81d4be5b4ba8ecea8be", + "size": 143, + "instruction": "/bin/sh -c ln -fs /usr/local/bin/wget-lua /grab/wget-at" + } + ], + "os": "linux", + "os_features": null, + "os_version": null, + "size": 356557380, + "status": "active", + "last_pulled": "2021-02-22T11:15:51.896128Z", + "last_pushed": "2020-10-31T11:04:01.777068Z" + } +] \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.info.json b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.info.json new file mode 100644 index 0000000000..1a7e201447 --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/repositories/youtube-playlistnotes-grab/tags/latest.info.json @@ -0,0 +1,30 @@ +{ + "creator": 6911312, + "id": 119983947, + "image_id": null, + "images": [ + { + "architecture": "amd64", + "features": "", + "variant": null, + "digest": "sha256:9893037e4182af8654f811f2802888178985cc21dcf2e5189a8e7672fbbda3c3", + "os": "linux", + "os_features": "", + "os_version": null, + "size": 356557380, + "status": "active", + "last_pulled": "2021-02-22T11:15:51.896128Z", + "last_pushed": "2020-10-31T11:04:01.777068Z" + } + ], + "last_updated": "2020-10-31T11:04:01.777068Z", + "last_updater": 6911312, + "last_updater_username": "warcforceone", + "name": "latest", + "repository": 9961945, + "full_size": 356557380, + "v2": true, + "tag_status": "active", + "tag_last_pulled": "2021-02-22T11:15:51.896128Z", + "tag_last_pushed": "2020-10-31T11:04:01.777068Z" +} \ No newline at end of file diff --git a/.tmp.7qbM4g7gLs/warcforceone/user.json b/.tmp.7qbM4g7gLs/warcforceone/user.json new file mode 100644 index 0000000000..3f41e7dc9d --- /dev/null +++ b/.tmp.7qbM4g7gLs/warcforceone/user.json @@ -0,0 +1,12 @@ +{ + "id": "834cf58376f642088956473cac0d4135", + "orgname": "warcforceone", + "full_name": "WARC Force One", + "location": "", + "company": "Archive Team", + "profile_url": "", + "date_joined": "2019-07-06T22:38:01.656830Z", + "gravatar_url": "https://secure.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e.jpg?s=80&r=g&d=mm", + "gravatar_email": "", + "type": "Organization" +} \ No newline at end of file diff --git a/archiveteam/repositories.json b/archiveteam/repositories.json index c0f82e799e..d10c01f015 100644 --- a/archiveteam/repositories.json +++ b/archiveteam/repositories.json @@ -10,7 +10,7 @@ "is_automated": true, "can_edit": false, "star_count": 16, - "pull_count": 10956097, + "pull_count": 10956449, "last_updated": "2018-12-17T00:42:59.374566Z", "is_migrated": true, "collaborator_count": 0, diff --git a/archiveteam/repositories/warrior-dockerfile/info.json b/archiveteam/repositories/warrior-dockerfile/info.json index 6b5bc7c501..db8cbfad66 100644 --- a/archiveteam/repositories/warrior-dockerfile/info.json +++ b/archiveteam/repositories/warrior-dockerfile/info.json @@ -9,7 +9,7 @@ "is_automated": true, "can_edit": false, "star_count": 16, - "pull_count": 10956097, + "pull_count": 10956450, "last_updated": "2018-12-17T00:42:59.374566Z", "is_migrated": true, "collaborator_count": 0, diff --git a/archiveteam/repositories/warrior-dockerfile/tags.json b/archiveteam/repositories/warrior-dockerfile/tags.json index 14bd38c03d..1cb03daf81 100644 --- a/archiveteam/repositories/warrior-dockerfile/tags.json +++ b/archiveteam/repositories/warrior-dockerfile/tags.json @@ -14,7 +14,7 @@ "os_version": null, "size": 119940149, "status": "active", - "last_pulled": "2021-03-03T00:03:51.273512Z", + "last_pulled": "2021-03-03T02:03:58.249211Z", "last_pushed": null } ], @@ -26,7 +26,7 @@ "full_size": 119940149, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-03-03T00:03:51.273512Z", + "tag_last_pulled": "2021-03-03T02:03:58.249211Z", "tag_last_pushed": "2018-12-17T00:42:59.092382Z" } ] \ No newline at end of file diff --git a/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json b/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json index 12ffdfafd6..143e6600fe 100644 --- a/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json +++ b/archiveteam/repositories/warrior-dockerfile/tags/latest.images.json @@ -164,7 +164,7 @@ "os_version": null, "size": 119940149, "status": "active", - "last_pulled": "2021-03-03T00:03:51.273512Z", + "last_pulled": "2021-03-03T02:03:58.249211Z", "last_pushed": null } ] \ No newline at end of file diff --git a/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json b/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json index e8d58aaae8..bd06d3af6b 100644 --- a/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json +++ b/archiveteam/repositories/warrior-dockerfile/tags/latest.info.json @@ -13,7 +13,7 @@ "os_version": null, "size": 119940149, "status": "active", - "last_pulled": "2021-03-03T00:03:51.273512Z", + "last_pulled": "2021-03-03T02:03:58.249211Z", "last_pushed": null } ], @@ -25,6 +25,6 @@ "full_size": 119940149, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-03-03T00:03:51.273512Z", + "tag_last_pulled": "2021-03-03T02:03:58.249211Z", "tag_last_pushed": "2018-12-17T00:42:59.092382Z" } \ No newline at end of file diff --git a/dockerhub-metadata.retrieve.log b/dockerhub-metadata.retrieve.log index 8a8e379643..311aec3d33 100644 --- a/dockerhub-metadata.retrieve.log +++ b/dockerhub-metadata.retrieve.log @@ -1,234 +1,234 @@ -2021-03-03T01:00:02.141843Z Retrieving Docker Hub user archiveteam -2021-03-03T01:00:02.142088Z Fetching https://hub.docker.com/v2/users/archiveteam/ -2021-03-03T01:00:03.056527Z Fetching https://hub.docker.com/v2/repositories/archiveteam/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:03.833170Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/ -2021-03-03T01:00:04.464438Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/dockerfile/ -2021-03-03T01:00:05.055693Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:05.951096Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:06.526035Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/ -2021-03-03T01:00:07.114548Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/images -2021-03-03T01:00:07.158137Z Done with archiveteam -2021-03-03T01:00:07.158718Z Retrieving Docker Hub user warcforceone -2021-03-03T01:00:07.158835Z Fetching https://hub.docker.com/v2/users/warcforceone/ -2021-03-03T01:00:07.982647Z Fetching https://hub.docker.com/v2/repositories/warcforceone/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:08.534217Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/ -2021-03-03T01:00:09.172980Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/dockerfile/ -2021-03-03T01:00:09.772134Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:10.665436Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:11.256870Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/ -2021-03-03T01:00:11.841608Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/images -2021-03-03T01:00:12.415054Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/ -2021-03-03T01:00:13.107046Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/dockerfile/ -2021-03-03T01:00:13.709523Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:14.723654Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:15.303397Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/ -2021-03-03T01:00:15.875902Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/images -2021-03-03T01:00:16.445169Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/ -2021-03-03T01:00:17.067905Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/dockerfile/ -2021-03-03T01:00:17.683568Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:18.799623Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:19.376876Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/ -2021-03-03T01:00:19.951885Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/images -2021-03-03T01:00:20.543392Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/ -2021-03-03T01:00:21.161490Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/dockerfile/ -2021-03-03T01:00:21.777070Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:22.645592Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:23.228997Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/ -2021-03-03T01:00:23.827697Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/images -2021-03-03T01:00:24.414400Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/ -2021-03-03T01:00:25.037072Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/dockerfile/ -2021-03-03T01:00:25.640917Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:26.533136Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:27.115592Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/ -2021-03-03T01:00:27.777834Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/images -2021-03-03T01:00:28.361517Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/ -2021-03-03T01:00:29.005710Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/dockerfile/ -2021-03-03T01:00:29.581417Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:30.510943Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:31.086635Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/ -2021-03-03T01:00:31.685590Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/images -2021-03-03T01:00:32.261219Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/ -2021-03-03T01:00:32.898433Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/dockerfile/ -2021-03-03T01:00:33.480912Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:34.450725Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:35.040895Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/ -2021-03-03T01:00:35.624635Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/images -2021-03-03T01:00:36.244693Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/ -2021-03-03T01:00:36.871427Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/dockerfile/ -2021-03-03T01:00:37.470607Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:38.357842Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:38.941970Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/ -2021-03-03T01:00:39.529911Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/images -2021-03-03T01:00:40.107895Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/ -2021-03-03T01:00:40.736962Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/dockerfile/ -2021-03-03T01:00:41.353992Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:42.322487Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:42.905097Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/ -2021-03-03T01:00:43.475128Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/images -2021-03-03T01:00:44.067488Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/ -2021-03-03T01:00:44.703898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/dockerfile/ -2021-03-03T01:00:45.303914Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:46.176939Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:46.772710Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/ -2021-03-03T01:00:47.338791Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/images -2021-03-03T01:00:47.926875Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/ -2021-03-03T01:00:48.533142Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/dockerfile/ -2021-03-03T01:00:49.118118Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:49.989420Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:50.565316Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/ -2021-03-03T01:00:51.132286Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/images -2021-03-03T01:00:51.726287Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/ -2021-03-03T01:00:52.364684Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/dockerfile/ -2021-03-03T01:00:52.963934Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:53.899398Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:54.492818Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/ -2021-03-03T01:00:55.063489Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/images -2021-03-03T01:00:55.635044Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/ -2021-03-03T01:00:56.259682Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/dockerfile/ -2021-03-03T01:00:56.859919Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:00:57.742805Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:00:58.319217Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/ -2021-03-03T01:00:58.893774Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/images -2021-03-03T01:00:59.474481Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/ -2021-03-03T01:01:00.089561Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/dockerfile/ -2021-03-03T01:01:00.675953Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:01.510213Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:02.088472Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/ -2021-03-03T01:01:02.680004Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/images -2021-03-03T01:01:03.256680Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/ -2021-03-03T01:01:03.890889Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/dockerfile/ -2021-03-03T01:01:04.493668Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:05.472296Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:06.059003Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/ -2021-03-03T01:01:06.697623Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/images -2021-03-03T01:01:07.278305Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/ -2021-03-03T01:01:08.262477Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/dockerfile/ -2021-03-03T01:01:08.857658Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:09.483433Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:10.208801Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/ -2021-03-03T01:01:10.795848Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/images -2021-03-03T01:01:11.388652Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/ -2021-03-03T01:01:12.006245Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/dockerfile/ -2021-03-03T01:01:12.601812Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:13.591717Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:14.186166Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/ -2021-03-03T01:01:14.758255Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/images -2021-03-03T01:01:15.357144Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/ -2021-03-03T01:01:16.007473Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/dockerfile/ -2021-03-03T01:01:16.599896Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:17.258790Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:17.871129Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/ -2021-03-03T01:01:18.444928Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/images -2021-03-03T01:01:19.023897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/ -2021-03-03T01:01:19.649267Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/dockerfile/ -2021-03-03T01:01:20.243896Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:21.144662Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:21.733098Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/ -2021-03-03T01:01:22.331148Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/images -2021-03-03T01:01:23.034557Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/ -2021-03-03T01:01:24.105043Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/dockerfile/ -2021-03-03T01:01:24.693580Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:25.559898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:26.139331Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/ -2021-03-03T01:01:26.737123Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/images -2021-03-03T01:01:27.338919Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/ -2021-03-03T01:01:27.971922Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/dockerfile/ -2021-03-03T01:01:28.568760Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:29.443586Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:30.053615Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/ -2021-03-03T01:01:30.643019Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/images -2021-03-03T01:01:31.222454Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/ -2021-03-03T01:01:31.851448Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/dockerfile/ -2021-03-03T01:01:32.452242Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:33.309109Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:33.896556Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/ -2021-03-03T01:01:34.491409Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/images -2021-03-03T01:01:35.067560Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/ -2021-03-03T01:01:35.708498Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/dockerfile/ -2021-03-03T01:01:36.297814Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:37.179907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:37.770145Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/ -2021-03-03T01:01:38.358155Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/images -2021-03-03T01:01:38.934816Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/ -2021-03-03T01:01:39.574224Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/dockerfile/ -2021-03-03T01:01:40.166849Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:40.798091Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:41.373193Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/ -2021-03-03T01:01:42.011138Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/dockerfile/ -2021-03-03T01:01:42.601682Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:43.466813Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:44.048028Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/ -2021-03-03T01:01:44.634466Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/images -2021-03-03T01:01:45.203338Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/ -2021-03-03T01:01:45.812859Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/dockerfile/ -2021-03-03T01:01:46.397607Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:47.300800Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:47.902025Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/ -2021-03-03T01:01:48.498365Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/images -2021-03-03T01:01:49.071274Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/ -2021-03-03T01:01:49.691829Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/dockerfile/ -2021-03-03T01:01:50.291812Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:51.158694Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:51.755380Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/ -2021-03-03T01:01:52.323468Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/images -2021-03-03T01:01:52.931906Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/ -2021-03-03T01:01:53.570664Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/dockerfile/ -2021-03-03T01:01:54.177869Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:55.064927Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:55.655919Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/ -2021-03-03T01:01:56.236399Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/images -2021-03-03T01:01:56.824756Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/ -2021-03-03T01:01:57.464641Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/dockerfile/ -2021-03-03T01:01:58.052043Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:01:58.889921Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:01:59.468134Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/ -2021-03-03T01:02:00.037232Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/images -2021-03-03T01:02:00.617869Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/ -2021-03-03T01:02:01.261649Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/dockerfile/ -2021-03-03T01:02:01.843065Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:02.691678Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:03.286232Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/ -2021-03-03T01:02:03.875729Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/images -2021-03-03T01:02:04.460285Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/ -2021-03-03T01:02:05.086547Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/dockerfile/ -2021-03-03T01:02:05.687904Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:06.625052Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:07.207903Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/ -2021-03-03T01:02:07.788868Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/images -2021-03-03T01:02:08.379497Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/ -2021-03-03T01:02:09.019896Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/dockerfile/ -2021-03-03T01:02:09.621986Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:10.534326Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:11.128838Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/ -2021-03-03T01:02:11.715060Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/images -2021-03-03T01:02:12.294140Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/ -2021-03-03T01:02:12.918611Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/dockerfile/ -2021-03-03T01:02:13.523658Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:14.390488Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:14.960577Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/ -2021-03-03T01:02:15.532563Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/images -2021-03-03T01:02:16.115907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/ -2021-03-03T01:02:16.751915Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/dockerfile/ -2021-03-03T01:02:17.339895Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:18.211725Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:18.794985Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/ -2021-03-03T01:02:19.378220Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/images -2021-03-03T01:02:19.954159Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/ -2021-03-03T01:02:20.602921Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/dockerfile/ -2021-03-03T01:02:21.199042Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:22.156628Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:22.741010Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/ -2021-03-03T01:02:23.318611Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/images -2021-03-03T01:02:23.919311Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/ -2021-03-03T01:02:24.541808Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/dockerfile/ -2021-03-03T01:02:25.139287Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:26.246741Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:26.849991Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/ -2021-03-03T01:02:27.425718Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/images -2021-03-03T01:02:28.005697Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/ -2021-03-03T01:02:28.626920Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/dockerfile/ -2021-03-03T01:02:29.219371Z Fetching https://hub.docker.com/api/build/v1/source/ -2021-03-03T01:02:30.118883Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/?page_size=100&page=1&ordering=last_updated -2021-03-03T01:02:30.698883Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/ -2021-03-03T01:02:31.279060Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/images -2021-03-03T01:02:31.316114Z Done with warcforceone +2021-03-03T03:00:01.277730Z Retrieving Docker Hub user archiveteam +2021-03-03T03:00:01.277934Z Fetching https://hub.docker.com/v2/users/archiveteam/ +2021-03-03T03:00:01.877708Z Fetching https://hub.docker.com/v2/repositories/archiveteam/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:03.283905Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/ +2021-03-03T03:00:05.304088Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/dockerfile/ +2021-03-03T03:00:05.937044Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:07.265828Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:07.865348Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/ +2021-03-03T03:00:08.486533Z Fetching https://hub.docker.com/v2/repositories/archiveteam/warrior-dockerfile/tags/latest/images +2021-03-03T03:00:08.534868Z Done with archiveteam +2021-03-03T03:00:08.535989Z Retrieving Docker Hub user warcforceone +2021-03-03T03:00:08.536190Z Fetching https://hub.docker.com/v2/users/warcforceone/ +2021-03-03T03:00:09.403004Z Fetching https://hub.docker.com/v2/repositories/warcforceone/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:09.932848Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/ +2021-03-03T03:00:10.549550Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/dockerfile/ +2021-03-03T03:00:11.288782Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:12.391544Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:12.982957Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/ +2021-03-03T03:00:13.555362Z Fetching https://hub.docker.com/v2/repositories/warcforceone/dnsmasq/tags/latest/images +2021-03-03T03:00:14.128768Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/ +2021-03-03T03:00:14.794827Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/dockerfile/ +2021-03-03T03:00:15.395906Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:16.399912Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:16.987006Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/ +2021-03-03T03:00:17.563235Z Fetching https://hub.docker.com/v2/repositories/warcforceone/angelfire-grab/tags/latest/images +2021-03-03T03:00:18.179910Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/ +2021-03-03T03:00:18.907621Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/dockerfile/ +2021-03-03T03:00:19.544768Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:20.425434Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:20.999904Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/ +2021-03-03T03:00:21.577960Z Fetching https://hub.docker.com/v2/repositories/warcforceone/grab-base/tags/latest/images +2021-03-03T03:00:22.149769Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/ +2021-03-03T03:00:22.867438Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/dockerfile/ +2021-03-03T03:00:23.471900Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:24.687911Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:25.264755Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/ +2021-03-03T03:00:25.837166Z Fetching https://hub.docker.com/v2/repositories/warcforceone/google-sites-grab/tags/latest/images +2021-03-03T03:00:26.414421Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/ +2021-03-03T03:00:27.123980Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/dockerfile/ +2021-03-03T03:00:27.713530Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:28.702162Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:29.285540Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/ +2021-03-03T03:00:29.864566Z Fetching https://hub.docker.com/v2/repositories/warcforceone/github-grab/tags/latest/images +2021-03-03T03:00:30.449132Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/ +2021-03-03T03:00:31.051907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/dockerfile/ +2021-03-03T03:00:31.642111Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:32.579899Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:33.159831Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/ +2021-03-03T03:00:33.749989Z Fetching https://hub.docker.com/v2/repositories/warcforceone/pastebin-grab/tags/latest/images +2021-03-03T03:00:34.331899Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/ +2021-03-03T03:00:35.093568Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/dockerfile/ +2021-03-03T03:00:35.692778Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:37.156658Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:37.723689Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/ +2021-03-03T03:00:38.303695Z Fetching https://hub.docker.com/v2/repositories/warcforceone/reddit-grab/tags/latest/images +2021-03-03T03:00:38.873978Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/ +2021-03-03T03:00:39.592636Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/dockerfile/ +2021-03-03T03:00:40.183789Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:41.046067Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:41.624575Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/ +2021-03-03T03:00:42.204520Z Fetching https://hub.docker.com/v2/repositories/warcforceone/domains-grab/tags/latest/images +2021-03-03T03:00:42.792916Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/ +2021-03-03T03:00:43.502114Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/dockerfile/ +2021-03-03T03:00:44.090468Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:45.278866Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:45.859883Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/ +2021-03-03T03:00:46.436652Z Fetching https://hub.docker.com/v2/repositories/warcforceone/megawarc-factory/tags/latest/images +2021-03-03T03:00:47.003603Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/ +2021-03-03T03:00:47.726934Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/dockerfile/ +2021-03-03T03:00:48.315564Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:49.282998Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:49.863916Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/ +2021-03-03T03:00:50.441384Z Fetching https://hub.docker.com/v2/repositories/warcforceone/fotoalbum-grab/tags/latest/images +2021-03-03T03:00:51.009432Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/ +2021-03-03T03:00:51.638670Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/dockerfile/ +2021-03-03T03:00:52.228464Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:53.087189Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:53.661069Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/ +2021-03-03T03:00:54.243757Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-playlistnotes-grab/tags/latest/images +2021-03-03T03:00:54.818087Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/ +2021-03-03T03:00:55.447032Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/dockerfile/ +2021-03-03T03:00:56.047771Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:00:56.594153Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:00:57.172848Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/ +2021-03-03T03:00:57.753515Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mercurial-grab/tags/latest/images +2021-03-03T03:00:58.325368Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/ +2021-03-03T03:00:58.944775Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/dockerfile/ +2021-03-03T03:00:59.535909Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:00.444805Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:01.015902Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/ +2021-03-03T03:01:01.611532Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tencent-weibo-grab/tags/latest/images +2021-03-03T03:01:02.189116Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/ +2021-03-03T03:01:02.835184Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/dockerfile/ +2021-03-03T03:01:03.436471Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:04.903897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:05.489083Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/ +2021-03-03T03:01:06.083710Z Fetching https://hub.docker.com/v2/repositories/warcforceone/theartistunion-grab/tags/latest/images +2021-03-03T03:01:06.653313Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/ +2021-03-03T03:01:07.652344Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/dockerfile/ +2021-03-03T03:01:08.266277Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:09.125503Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:09.709680Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/ +2021-03-03T03:01:10.282060Z Fetching https://hub.docker.com/v2/repositories/warcforceone/bitbucket-grab/tags/latest/images +2021-03-03T03:01:10.850656Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/ +2021-03-03T03:01:11.478228Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/dockerfile/ +2021-03-03T03:01:12.069735Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:12.692395Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:13.269911Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/ +2021-03-03T03:01:13.839719Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync/tags/latest/images +2021-03-03T03:01:14.410746Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/ +2021-03-03T03:01:15.054638Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/dockerfile/ +2021-03-03T03:01:15.649423Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:16.562143Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:17.143907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/ +2021-03-03T03:01:17.724463Z Fetching https://hub.docker.com/v2/repositories/warcforceone/naver-matome-grab/tags/latest/images +2021-03-03T03:01:18.308706Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/ +2021-03-03T03:01:18.975900Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/dockerfile/ +2021-03-03T03:01:19.566562Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:20.175876Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:20.750130Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/ +2021-03-03T03:01:21.331626Z Fetching https://hub.docker.com/v2/repositories/warcforceone/ateam-airsync-meta/tags/latest/images +2021-03-03T03:01:22.115685Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/ +2021-03-03T03:01:22.742586Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/dockerfile/ +2021-03-03T03:01:23.330593Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:24.194524Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:24.772985Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/ +2021-03-03T03:01:25.371512Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutch-grab/tags/latest/images +2021-03-03T03:01:25.947897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/ +2021-03-03T03:01:26.567913Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/dockerfile/ +2021-03-03T03:01:27.159967Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:28.064900Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:28.638081Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/ +2021-03-03T03:01:29.233159Z Fetching https://hub.docker.com/v2/repositories/warcforceone/clutchvideos-grab/tags/latest/images +2021-03-03T03:01:29.813554Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/ +2021-03-03T03:01:30.451897Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/dockerfile/ +2021-03-03T03:01:31.042283Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:31.999640Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:32.587351Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/ +2021-03-03T03:01:33.182531Z Fetching https://hub.docker.com/v2/repositories/warcforceone/microsoft-download-center-grab/tags/latest/images +2021-03-03T03:01:33.763371Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/ +2021-03-03T03:01:34.399365Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/dockerfile/ +2021-03-03T03:01:34.986173Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:35.949525Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:36.523909Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/ +2021-03-03T03:01:37.091909Z Fetching https://hub.docker.com/v2/repositories/warcforceone/mixer-grab/tags/latest/images +2021-03-03T03:01:37.662926Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/ +2021-03-03T03:01:38.265421Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/dockerfile/ +2021-03-03T03:01:38.859864Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:39.851964Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:40.432730Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/ +2021-03-03T03:01:41.019092Z Fetching https://hub.docker.com/v2/repositories/warcforceone/soup-io-grab/tags/latest/images +2021-03-03T03:01:41.598898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/ +2021-03-03T03:01:42.221466Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/dockerfile/ +2021-03-03T03:01:42.808187Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:43.466805Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape-upstream/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:44.041894Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/ +2021-03-03T03:01:44.682935Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/dockerfile/ +2021-03-03T03:01:45.271700Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:46.177752Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:46.753480Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/ +2021-03-03T03:01:47.332440Z Fetching https://hub.docker.com/v2/repositories/warcforceone/snscrape/tags/latest/images +2021-03-03T03:01:47.900701Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/ +2021-03-03T03:01:48.508712Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/dockerfile/ +2021-03-03T03:01:49.095896Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:50.055459Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:50.624713Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/ +2021-03-03T03:01:51.215755Z Fetching https://hub.docker.com/v2/repositories/warcforceone/singstar-grab/tags/latest/images +2021-03-03T03:01:51.785615Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/ +2021-03-03T03:01:52.395413Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/dockerfile/ +2021-03-03T03:01:52.981580Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:53.853315Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:54.424934Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/ +2021-03-03T03:01:55.007666Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yahoogroups-grab/tags/latest/images +2021-03-03T03:01:55.586794Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/ +2021-03-03T03:01:56.224635Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/dockerfile/ +2021-03-03T03:01:56.823295Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:01:57.667364Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:01:58.247659Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/ +2021-03-03T03:01:58.831906Z Fetching https://hub.docker.com/v2/repositories/warcforceone/playstv-grab/tags/latest/images +2021-03-03T03:01:59.407768Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/ +2021-03-03T03:02:00.270645Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/dockerfile/ +2021-03-03T03:02:00.858736Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:01.831193Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:02.403903Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/ +2021-03-03T03:02:02.986967Z Fetching https://hub.docker.com/v2/repositories/warcforceone/youtube-likedlists-grab/tags/latest/images +2021-03-03T03:02:03.563509Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/ +2021-03-03T03:02:04.176700Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/dockerfile/ +2021-03-03T03:02:04.780512Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:05.647903Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:06.227270Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/ +2021-03-03T03:02:06.815835Z Fetching https://hub.docker.com/v2/repositories/warcforceone/gfycat-grab/tags/latest/images +2021-03-03T03:02:07.389941Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/ +2021-03-03T03:02:08.011904Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/dockerfile/ +2021-03-03T03:02:08.603663Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:09.526505Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:10.097526Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/ +2021-03-03T03:02:10.667285Z Fetching https://hub.docker.com/v2/repositories/warcforceone/drawr-grab/tags/latest/images +2021-03-03T03:02:11.249595Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/ +2021-03-03T03:02:11.911320Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/dockerfile/ +2021-03-03T03:02:12.501368Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:13.389500Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:13.963909Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/ +2021-03-03T03:02:14.537432Z Fetching https://hub.docker.com/v2/repositories/warcforceone/radio24syv-grab/tags/latest/images +2021-03-03T03:02:15.110366Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/ +2021-03-03T03:02:15.727607Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/dockerfile/ +2021-03-03T03:02:16.318022Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:17.195723Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:17.791411Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/ +2021-03-03T03:02:18.382037Z Fetching https://hub.docker.com/v2/repositories/warcforceone/yourshot-grab/tags/latest/images +2021-03-03T03:02:18.968563Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/ +2021-03-03T03:02:19.596934Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/dockerfile/ +2021-03-03T03:02:20.266673Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:21.205479Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:21.785661Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/ +2021-03-03T03:02:22.374532Z Fetching https://hub.docker.com/v2/repositories/warcforceone/sketch-grab/tags/latest/images +2021-03-03T03:02:22.950919Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/ +2021-03-03T03:02:23.567907Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/dockerfile/ +2021-03-03T03:02:24.171450Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:25.254492Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:25.839898Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/ +2021-03-03T03:02:26.413479Z Fetching https://hub.docker.com/v2/repositories/warcforceone/tinypic-grab/tags/latest/images +2021-03-03T03:02:26.988776Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/ +2021-03-03T03:02:27.611514Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/dockerfile/ +2021-03-03T03:02:28.194212Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:29.046678Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:29.619892Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/ +2021-03-03T03:02:30.201887Z Fetching https://hub.docker.com/v2/repositories/warcforceone/freeml-grab/tags/latest/images +2021-03-03T03:02:30.781764Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/ +2021-03-03T03:02:31.390088Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/dockerfile/ +2021-03-03T03:02:31.975863Z Fetching https://hub.docker.com/api/build/v1/source/ +2021-03-03T03:02:32.816688Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/?page_size=100&page=1&ordering=last_updated +2021-03-03T03:02:33.392816Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/ +2021-03-03T03:02:33.969551Z Fetching https://hub.docker.com/v2/repositories/warcforceone/instaudio-grab/tags/latest/images +2021-03-03T03:02:34.005506Z Done with warcforceone diff --git a/warcforceone/repositories.json b/warcforceone/repositories.json index 33e3465c81..301768d4cb 100644 --- a/warcforceone/repositories.json +++ b/warcforceone/repositories.json @@ -10,7 +10,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 21108852, + "pull_count": 21109332, "last_updated": "2021-02-20T00:18:15.003220Z", "is_migrated": false, "collaborator_count": 0, @@ -82,7 +82,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 13991560, + "pull_count": 13993910, "last_updated": "2021-02-03T13:48:21.932838Z", "is_migrated": false, "collaborator_count": 0, diff --git a/warcforceone/repositories/dnsmasq/info.json b/warcforceone/repositories/dnsmasq/info.json index 168aa41890..dc9545bb51 100644 --- a/warcforceone/repositories/dnsmasq/info.json +++ b/warcforceone/repositories/dnsmasq/info.json @@ -9,7 +9,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 21108852, + "pull_count": 21109332, "last_updated": "2021-02-20T00:18:15.003220Z", "is_migrated": false, "collaborator_count": 0, diff --git a/warcforceone/repositories/dnsmasq/tags.json b/warcforceone/repositories/dnsmasq/tags.json index f899de0a03..3b227599ad 100644 --- a/warcforceone/repositories/dnsmasq/tags.json +++ b/warcforceone/repositories/dnsmasq/tags.json @@ -14,7 +14,7 @@ "os_version": null, "size": 4477304, "status": "active", - "last_pulled": "2021-03-03T00:56:51.459619Z", + "last_pulled": "2021-03-03T02:56:52.153918Z", "last_pushed": "2021-02-20T00:18:14.704711Z" } ], @@ -26,7 +26,7 @@ "full_size": 4477304, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-03-03T00:56:51.459619Z", + "tag_last_pulled": "2021-03-03T02:56:52.153918Z", "tag_last_pushed": "2021-02-20T00:18:14.704711Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/dnsmasq/tags/latest.images.json b/warcforceone/repositories/dnsmasq/tags/latest.images.json index 55f1c9488f..9a674d4e8e 100644 --- a/warcforceone/repositories/dnsmasq/tags/latest.images.json +++ b/warcforceone/repositories/dnsmasq/tags/latest.images.json @@ -38,7 +38,7 @@ "os_version": null, "size": 4477304, "status": "active", - "last_pulled": "2021-03-03T00:56:51.459619Z", + "last_pulled": "2021-03-03T02:56:52.153918Z", "last_pushed": "2021-02-20T00:18:14.704711Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/dnsmasq/tags/latest.info.json b/warcforceone/repositories/dnsmasq/tags/latest.info.json index 1440a74683..443620310f 100644 --- a/warcforceone/repositories/dnsmasq/tags/latest.info.json +++ b/warcforceone/repositories/dnsmasq/tags/latest.info.json @@ -13,7 +13,7 @@ "os_version": null, "size": 4477304, "status": "active", - "last_pulled": "2021-03-03T00:56:51.459619Z", + "last_pulled": "2021-03-03T02:56:52.153918Z", "last_pushed": "2021-02-20T00:18:14.704711Z" } ], @@ -25,6 +25,6 @@ "full_size": 4477304, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-03-03T00:56:51.459619Z", + "tag_last_pulled": "2021-03-03T02:56:52.153918Z", "tag_last_pushed": "2021-02-20T00:18:14.704711Z" } \ No newline at end of file diff --git a/warcforceone/repositories/github-grab/info.json b/warcforceone/repositories/github-grab/info.json index 75f9ec6a02..6db7ed584e 100644 --- a/warcforceone/repositories/github-grab/info.json +++ b/warcforceone/repositories/github-grab/info.json @@ -9,7 +9,7 @@ "is_automated": false, "can_edit": false, "star_count": 0, - "pull_count": 13991567, + "pull_count": 13993914, "last_updated": "2021-02-03T13:48:21.932838Z", "is_migrated": false, "collaborator_count": 0, diff --git a/warcforceone/repositories/github-grab/tags.json b/warcforceone/repositories/github-grab/tags.json index d97e10f146..effd7c3913 100644 --- a/warcforceone/repositories/github-grab/tags.json +++ b/warcforceone/repositories/github-grab/tags.json @@ -14,7 +14,7 @@ "os_version": null, "size": 355285549, "status": "active", - "last_pulled": "2021-03-03T00:34:54.542367Z", + "last_pulled": "2021-03-03T02:34:55.780991Z", "last_pushed": "2021-02-03T13:48:21.641084Z" } ], @@ -26,7 +26,7 @@ "full_size": 355285549, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-03-03T00:34:54.542367Z", + "tag_last_pulled": "2021-03-03T02:34:55.780991Z", "tag_last_pushed": "2021-02-03T13:48:21.641084Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/github-grab/tags/latest.images.json b/warcforceone/repositories/github-grab/tags/latest.images.json index b3163de781..5b8a8fbf09 100644 --- a/warcforceone/repositories/github-grab/tags/latest.images.json +++ b/warcforceone/repositories/github-grab/tags/latest.images.json @@ -148,7 +148,7 @@ "os_version": null, "size": 355285549, "status": "active", - "last_pulled": "2021-03-03T00:34:54.542367Z", + "last_pulled": "2021-03-03T02:34:55.780991Z", "last_pushed": "2021-02-03T13:48:21.641084Z" } ] \ No newline at end of file diff --git a/warcforceone/repositories/github-grab/tags/latest.info.json b/warcforceone/repositories/github-grab/tags/latest.info.json index 4f0fe280c5..e8696f18f6 100644 --- a/warcforceone/repositories/github-grab/tags/latest.info.json +++ b/warcforceone/repositories/github-grab/tags/latest.info.json @@ -13,7 +13,7 @@ "os_version": null, "size": 355285549, "status": "active", - "last_pulled": "2021-03-03T00:34:54.542367Z", + "last_pulled": "2021-03-03T02:34:55.780991Z", "last_pushed": "2021-02-03T13:48:21.641084Z" } ], @@ -25,6 +25,6 @@ "full_size": 355285549, "v2": true, "tag_status": "active", - "tag_last_pulled": "2021-03-03T00:34:54.542367Z", + "tag_last_pulled": "2021-03-03T02:34:55.780991Z", "tag_last_pushed": "2021-02-03T13:48:21.641084Z" } \ No newline at end of file