re-initialize and make some changes. repository got to around 2 gigabytes and needed to be more efficient

.domains

@@ -0,0 +1,4 @@
+hrt.boytw.ink
+diyhrt.codeberg.page
+pages.diyhrt.codeberg.page
+pages.pages.diyhrt.codeberg.page

.github/workflows/codeberg_mirror.yml

@@ -0,0 +1,38 @@
+name: Mirror to Codeberg
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  codeberg:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: check for 404
+        run: |
+          url="https://codeberg.org/diyhrt/pages"
+          status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+          if [ $status -eq 404 ]; then
+            echo "Got 404. Exiting."
+            curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
+            exit 1
+          fi
+      - uses: pixta-dev/repository-mirroring-action@v1.1.1
+        with:
+          target_repo_url: "git@codeberg.org:diyhrt/pages.git"
+          ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
+  secret_mirror:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.PAT_TOKEN }}
+      - uses: pixta-dev/repository-mirroring-action@v1.1.1
+        with:
+          target_repo_url: "${{ secrets.SECRET_MIRROR_URL }}"
+          ssh_private_key: ${{ secrets.SECRET_SSH_PRIVATE_KEY }}

.github/workflows/disroot_mirror.yml

@@ -0,0 +1,28 @@
+name: Mirror to Disroot
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 1,15 * *"
+
+jobs:
+  disroot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: check for 404
+        run: |
+          url="https://git.disroot.org/diyhrt/mirrors"
+          status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+          if [ $status -eq 404 ]; then
+            echo "Got 404. Exiting."
+            curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
+            exit 1
+          fi
+      - uses: pixta-dev/repository-mirroring-action@v1.1.1
+        with:
+          target_repo_url: "git@git.disroot.org:diyhrt/mirrors.git"
+          ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

.github/workflows/diyhrt.cafe.yml

@@ -0,0 +1,36 @@
+name: DIYHRT.cafe
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 0"
+
+jobs:
+  download:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: check for 404
+        run: |
+          url="https://diyhrt.cafe"
+          status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+          if [ $status -eq 404 ]; then
+            echo "Got 404. Exiting."
+            curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
+            exit 1
+          fi
+      - name: download .cafe
+        if: always()
+        run: |
+          rm -rf diyhrt.cafe
+          wget --convert-links --page-requisites --span-hosts --no-parent --reject 'rocket-loader.min.js' --reject '*UserLogin*' --reject '*visualEditor*' --reject '*CreateAccount*' --reject '*load.php*' --wait 3 --limit-rate 4m --continue --random-wait --adjust-extension -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4 Contact: hrt.boytw.ink/crawl_contact.txt' -e robots=off -i scripts/cafe.txt || true
+      - name: commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "diyhrt.cafe"
+          pull: "--rebase --autostash ."
+          message: "[automated] update diyhrt.cafe"
+          default_author: github_actions

.github/workflows/diyhrt.market.yml

@@ -0,0 +1,36 @@
+name: DIYHRT.market
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 1"
+
+jobs:
+  archive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: check for 404
+        run: |
+          url="https://diyhrt.market"
+          status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+          if [ $status -eq 404 ]; then
+            echo "Got 404. Exiting."
+            curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
+            exit 1
+          fi
+      - name: download diyhrt.market
+        run: |
+          rm -rf diyhrt.market cdn.diyhrt.market
+          wget -mkpKEH --limit-rate 10m --random-wait --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --domains=diyhrt.market,cdn.diyhrt.market  https://diyhrt.market || true
+          find . -name "*.orig" -type f -delete
+      - name: commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "['diyhrt.market', 'cdn.diyhrt.market']"
+          pull: "--rebase --autostash ."
+          message: "[automated] update diyhrt.market"
+          default_author: github_actions

.github/workflows/diyhrt.wiki.yml

@@ -0,0 +1,42 @@
+name: DIYHRT.wiki
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 2"
+
+jobs:
+  archive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: check for 404
+        run: |
+          url="https://diyhrt.wiki"
+          status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+          if [ $status -eq 404 ]; then
+            echo "Got 404. Exiting."
+            curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
+            exit 1
+          fi
+      - name: download diyhrt.wiki
+        run: |
+          rm -rf diyhrt.wiki
+          wget -mkxKE -e robots=off --limit-rate=10m --random-wait --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --reject '*age-check*' https://diyhrt.wiki https://diyhrt.wiki/nav.html || true # downloads nav.html because it doesn't crawl properly without it
+          cd diyhrt.wiki && rm *.orig && cd ..
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: run agebypass.py # removes the age-check.js script because it was causing problems
+        run: python scripts/agebypass.py
+      - name: commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "diyhrt.wiki"
+          pull: "--rebase --autostash ."
+          message: "[automated] update diyhrt.wiki"
+          default_author: github_actions

.github/workflows/groups.io.yml

@@ -0,0 +1,26 @@
+name: groups.io wiki
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 3"
+
+jobs:
+  archive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: download groups.io wiki
+        run: |
+          rm -rf groups.io
+          wget -e robots=off --mirror --limit-rate=10m --convert-links --adjust-extension --page-requisites --random-wait --no-parent https://groups.io/g/MTFHRT/wiki/ https://groups.io/g/MTFHRT/topic/80225302/ https://groups.io/g/MTFHRT/topic/79606271/ || true
+      - name: commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "groups.io"
+          pull: --rebase --autostash .
+          message: "[automated] update groups.io wiki"
+          default_author: github_actions

.github/workflows/hrt.coffee.yml

@@ -0,0 +1,40 @@
+name: HRT.coffee
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 4'
+
+jobs:
+  archive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: Check for 404
+        run: |
+          url="https://hrt.coffee"
+          status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+          if [ $status -eq 404 ]; then
+            echo "Got 404. Exiting."
+            curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
+            exit 1
+          fi
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install wget
+      - name: Download hrt.coffee
+        run: |
+          rm -rf hrt.coffee
+          export USER_AGENT="Mosaic/9.0 (Commodore AmigaOS 4.0) AmiWeb/4.1.33 (KHTML, like Gecko) Version/22.1 Mozilla/5.0. Contact: https://boytw.ink/crawl_contact.txt"
+          wget --adjust-extension --convert-links -e robots=off --mirror --page-requisites --waitretry 5 --timeout 60 --tries 5 --wait 1 -U "$USER_AGENT" https://hrt.coffee
+      - name: Commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: hrt.coffee
+          pull: "--rebase --autostash ."
+          message: "[automated] update hrt.coffee"
+          default_author: github_actions

.github/workflows/md_to_html.yml

@@ -0,0 +1,32 @@
+name: Convert markdown to html
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+    - 'README.md'
+    - 'DISCLAIMER.md'
+
+jobs:
+  convert:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: pypy-3.7-v7.3.3
+      - name: Convert to html
+        run: |
+          pip install markdown bleach
+          python scripts/md_to_html.py README.md readme.html
+          python scripts/md_to_html.py DISCLAIMER.md disclaimer.html
+      - name: Commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "['readme.html', 'disclaimer.html']"
+          pull: "--rebase --autostash ."
+          message: "[automated] update markdown files"
+          default_author: github_actions

.github/workflows/reddit.yml

@@ -0,0 +1,40 @@
+name: Reddit Wikis
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 5"
+
+jobs:
+  archive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: check for 404
+        run: |
+          urls=("https://old.reddit.com/r/TransDIY/wiki/pharmacies" "https://old.reddit.com/r/TransDIY/wiki" "https://old.reddit.com/r/TransSurgeriesWiki/wiki/index/" "https://old.reddit.com/r/FTM/wiki")
+          failed=""
+          for url in "${urls[@]}"; do
+            status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+            if [ $status -eq 404 ]; then
+              failed=$url
+              echo "Got 404 on $failed. Exiting."
+              curl -H "Content-Type: application/json" -d "{\"content\": \"${{ secrets.DISCORD_USER_ID }} $url returned 404\"}" ${{ secrets.WEBHOOK_URL }}
+            fi
+          done
+      - name: Download Wikis
+        run: |
+          rm -rf old.reddit.com && rm -rf reddit
+          wget --no-cookies --header "Cookie: over18=1" -E -H -k -p -r -l1 --domains=old.reddit.com https://old.reddit.com/r/TransDIY/wiki https://old.reddit.com/r/TransDIY/wiki/pharmacies https://old.reddit.com/r/TransSurgeriesWiki/wiki/index/ https://old.reddit.com/r/FTM/wiki || true
+          mv old.reddit.com reddit
+          cd reddit && rm robots.txt
+      - name: Commit Changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: reddit
+          pull: --rebase --autostash .
+          message: "[automated] update /r/TransDIY wiki"
+          default_author: github_actions

.github/workflows/transfemscience.org.yml

@@ -0,0 +1,36 @@
+name: transfemscience.org
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 6"
+
+jobs:
+  archive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - name: check for 404
+        run: |
+          url="https://transfemscience.org"
+          status=$(curl -o /dev/null -s -w "%{http_code}" $url)
+          if [ $status -eq 404 ]; then
+            echo "Got 404. Exiting."
+            curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
+            exit 1
+          fi
+      - name: download transfemscience
+        run: |
+          rm -rf transfemscience.org
+          wget -mkpKEH --limit-rate 10m --random-wait -e robots=off --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --domains=transfemscience.org,files.transfemscience.org https://transfemscience.org || true
+          find . -name "*.orig" -type f -delete
+      - name: commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "['files.transfemscience.org', 'transfemscience.org']"
+          pull: "--rebase --autostash ."
+          message: "[automated] update transfemscience"
+          default_author: github_actions

.github/workflows/update_cafe.yml

@@ -0,0 +1,32 @@
+name: Update DIYHRT.cafe links
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 23 1 * *" # every 1st of month 
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: pypy-3.7-v7.3.3
+      - name: Update links
+        if: always()
+        run: |
+          rm scripts/cafe.txt
+          pip install beautifulsoup4 bs4
+          python scripts/getlinks.py 'https://diyhrt.cafe/index.php/Special:AllPages' > scripts/cafe.txt
+          cd scripts && python formatlinks.py cafe.txt
+      - name: Commit changes
+        if: ${{ success() }}
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "scripts/cafe.txt"
+          pull: "--rebase --autostash ."
+          message: "[automated] update diyhrt.cafe list"
+          default_author: github_actions

404.html

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>404</title>
+  <style>
+    body {
+      background-color: black;
+      color: white;
+      text-align: center;
+      padding: 100px;
+      font-family: Arial, sans-serif;
+    }
+    a {
+      color: white;
+      text-decoration: underline;
+    }
+  </style>
+</head>
+<body>
+  <p>404 - if this is unexpected, please open an <a href="https://github.com/boytwink/diyhrt/issues">issue</a></p>
+</body>
+</html>

DISCLAIMER.md

@@ -0,0 +1,15 @@
+We cannot guarantee the legal status of self medication or ordering of medication online or importation of medication online in your country. We are not responsible for any damages whatsoever. All content on this website is exclusively for educational purposes. 
+
+All orders made with third party suppliers are at your own risk.
+
+Any information you may find on this website may be inaccurate, misleading, dangerous, unethical, or illegal.
+
+We make no effort remove any incorrect or dangerous information from this website. Use at your own risk, and by using it you agree that we are liable for nothing you do as a result of this website
+
+All information found on the site is without any implied warranty of fitness for any purpose or use whatsoever.
+
+Do not rely on content found on this website without independent verification.
+
+By using this website, you acknowledge that you have read this disclaimer and agree not to hold us responsible for anything you see or any action you take becase of this website.
+
+This website should not be accessed by anyone under the age of 21, and by using it you agree that you are 21 or over.

README.md

@@ -0,0 +1,18 @@
+## What is this?
+This an archive of diyhrt.cafe, diyhrt.wiki, diyhrt.market, hrt.coffee, transfemscience.org and a few others in case something goes wrong. Websites are updated once weekly.
+
+## Accessing this archive
+### Online, no download required.
+https://hrt.boytw.ink
+
+### Locally
+Use [this link](https://codeberg.org/diyhrt/pages/archive/main.zip) and it will start a download. To access the archive, open index.html with your browser.
+
+## Mirroring
+
+I encourage you to start your own mirror, and open an issue or pull request if you would like it added to the readme. If you decide create a public mirror, it should be automatically by simply running git pull at least once a week with a [simple cron job](https://stackoverflow.com/a/69553820). Feel free to open an issue if you need any help.
+
+This project is currently hosted on Codeberg and mirrored at [GitHub](https://boytwink.github.io/diyhrt) and [git.disroot.org](https://git.disroot.org/diyhrt/mirrors).
+
+## Contact me
+If you want to contact me about something here, start an issue on [GitHub](https://github.com/boytwink/diyhrt/issues) or [Codeberg](https://codeberg.org/diyhrt/pages/issues).

crawl_contact.txt

@@ -0,0 +1,2 @@
+if you're a webmaster who wants to contact me for any reason - open an issue github.com/boytwink/diyhrt/issues and give it the webmaster label. there's also a couple ways to contact me on my github profile.
+i do not and will not respect robots.txt. if you want me to not crawl a certain page or use different rate limits, read this https://wiki.archiveteam.org/index.php/Robots.txt, then ask me. 

disclaimer.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<!-- this file is automatically updated when README.md is updated, please don't modify it -->
+
+<head>
+    <title>readme</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="icon" type="image/png" href="/favicon.png">
+    <style>
+        body {
+            background-color: black;
+            color: white;
+            font-size: 20px;
+            font-family: sans-serif;
+        }
+
+        a {
+            color: white;
+        }
+    </style>
+</head>
+<body>
+<p>We cannot guarantee the legal status of self medication or ordering of medication online or importation of medication online in your country. We are not responsible for any damages whatsoever. All content on this website is exclusively for educational purposes. </p>
+<p>All orders made with third party suppliers are at your own risk.</p>
+<p>Any information you may find on this website may be inaccurate, misleading, dangerous, unethical, or illegal.</p>
+<p>We make no effort remove any incorrect or dangerous information from this website. Use at your own risk, and by using it you agree that we are liable for nothing you do as a result of this website</p>
+<p>All information found on the site is without any implied warranty of fitness for any purpose or use whatsoever.</p>
+<p>Do not rely on content found on this website without independent verification.</p>
+<p>By using this website, you acknowledge that you have read this disclaimer and agree not to hold us responsible for anything you see or any action you take becase of this website.</p>
+<p>This website should not be accessed by anyone under the age of 21, and by using it you agree that you are 21 or over.</p>
+</body>
+</html>

diyhrt.cafe/cdn-cgi/scripts/5c5dd728/cloudflare-static/email-decode.min.js

@@ -0,0 +1 @@
+!function(){"use strict";function e(e){try{if("undefined"==typeof console)return;"error"in console?console.error(e):console.log(e)}catch(e){}}function t(e){return d.innerHTML='<a href="'+e.replace(/"/g,"&quot;")+'"></a>',d.childNodes[0].getAttribute("href")||""}function r(e,t){var r=e.substr(t,2);return parseInt(r,16)}function n(n,c){for(var o="",a=r(n,c),i=c+2;i<n.length;i+=2){var l=r(n,i)^a;o+=String.fromCharCode(l)}try{o=decodeURIComponent(escape(o))}catch(u){e(u)}return t(o)}function c(t){for(var r=t.querySelectorAll("a"),c=0;c<r.length;c++)try{var o=r[c],a=o.href.indexOf(l);a>-1&&(o.href="mailto:"+n(o.href,a+l.length))}catch(i){e(i)}}function o(t){for(var r=t.querySelectorAll(u),c=0;c<r.length;c++)try{var o=r[c],a=o.parentNode,i=o.getAttribute(f);if(i){var l=n(i,0),d=document.createTextNode(l);a.replaceChild(d,o)}}catch(h){e(h)}}function a(t){for(var r=t.querySelectorAll("template"),n=0;n<r.length;n++)try{i(r[n].content)}catch(c){e(c)}}function i(t){try{c(t),o(t),a(t)}catch(r){e(r)}}var l="/cdn-cgi/l/email-protection#",u=".__cf_email__",f="data-cfemail",d=document.createElement("div");i(document),function(){var e=document.currentScript||document.scripts[document.scripts.length-1];e.parentNode.removeChild(e)}()}();