Initial commit

.github/workflows/deploy.yaml

@@ -0,0 +1,47 @@
+name: Push to prod
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch: # manual
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          known_hosts: ${{ secrets.KNOWN_HOSTS }}
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.x'
+          cache-dependency-path: |
+            go.sum
+      
+      - name: Fmt
+        run: make fmt
+
+      - name: Build
+        run: make build
+
+      - name: Test
+        run: make test
+
+      - name: Sync executables
+        run: |
+          rsync -avz "ingest" "serve" ${{ secrets.REMOTE_USER }}@${{ secrets.REMOTE_HOST }}:${{ secrets.REMOTE_PATH }}
+
+    
+      - name: Restart executables
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.REMOTE_HOST }}
+          username: ${{ secrets.REMOTE_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          script_path: .github/workflows/restart.sh

.github/workflows/deploy_site.yaml

@@ -0,0 +1,42 @@
+name: Sync Static Files
+
+on:
+  push:
+    paths:
+      - 'static/**'
+    branches:
+      - main
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get changed files in static directory
+        id: changed-files
+        uses: tj-actions/changed-files@v41
+        with:
+          files: static/**
+
+      - name: Install SSH key
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          known_hosts: ${{ secrets.KNOWN_HOSTS }}
+
+      - name: Sync files if changes detected
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+            rsync -avz --delete ./static/ ${{ secrets.REMOTE_USER }}@${{ secrets.REMOTE_HOST }}:${{ secrets.REMOTE_PATH }}/static/
+
+      - name: Restart executables
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.REMOTE_HOST }}
+          username: ${{ secrets.REMOTE_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          script_path: .github/workflows/restart.sh

.github/workflows/restart.sh

@@ -0,0 +1,15 @@
+cd /var/www/html
+echo "Changed directory to /var/www/html"
+
+# Only kill if processes exist
+pkill 'ingest'
+echo "Killed ingest processes"
+
+pkill 'serve'
+echo "Killed serve processes"
+
+nohup ./ingest > ingest.log 2>&1 &
+echo "Started ingest process"
+
+nohup ./serve > serve.log 2>&1 &
+echo "Started serve process"

.gitignore

@@ -0,0 +1,4 @@
+# sqlite
+*.db
+
+.env

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Vicki Boykis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,40 @@
+all: run
+.DEFAULT_GOAL := build
+
+fmt:
+	go fmt ./...
+.PHONY: fmt
+
+vet: fmt
+	go vet ./...
+.PHONY: vet
+
+build:
+	go build cmd/serve/serve.go
+	go build cmd/ingest/ingest.go
+.PHONY: build
+
+test:
+	go test ./...
+.PHONY: test
+
+run-serve:
+	CGO_ENABLED=1 go run cmd/serve/serve.go &
+.PHONY: run-serve
+
+run-ingest:
+	CGO_ENABLED=1 go run cmd/ingest/ingest.go &
+.PHONY: run-ingest
+
+kill-serve:
+	pkill -f "CGO_ENABLED=1 go run cmd/serve/serve.go" || true
+
+kill-ingest:
+	pkill -f "CGO_ENABLED=1 go run cmd/ingest/ingest.go" || true
+
+run: run-ingest run-serve
+.PHONY: run
+
+
+
+

README.md

@@ -0,0 +1,141 @@
+<img src="static/android-chrome-512x512.png" alt="drawing" width="400"/>
+
+# GitFeed
+
+A Go-based web app that collects all posts with GitHub links from the [Bluesky Jetstream](https://docs.bsky.app/blog/jetstream) by chronological order. It displays the last 10 links in a reverse-chronological stream so you can see what GitHub repos people are chatting about on Bluesky. There are about ~100 of these events per day, collected via websocket. 
+
+> [!NOTE]  
+> This app is experimental. Expect breaking API changes up until minor semver release. ⚡️ 
+
+<div style="text-align: center;"><img src="/ui.png" alt="drawing" width="400"/></div>
+
+## Running: 
+
+Gitfeed consists of a post ingest/delete process and a frontend. 
+
+1. `git clone`
+2.  Kill all instances of running application
+3.  Run application 
+    + `make run-serve`  # Runs the API and frontend
+    + `make run-ingest` # Runs the ingest from the jetstream
+
+## Developing:
+
+Gitfeed inclues a Go API that abstracts the repository pattern over a SQLite db. Code can be built and deployed using Go binaries. 
+The frontend is pure CSS/HTML/Javascript and can be developed ad-hoc without npm. 
+
+<div style="text-align: center;"><img src="/architecture.png" alt="drawing" width="400"/></div>
+
+To run both, build the Go binary to ingest the data and start the front-end to serve the app and the API. 
+
+1. `git clone`
+2. Backend lives in `cmd`, 'routes', `handlers`, and `db`
+3. Frontend lives in `static`
+4. Use makefile-based development: `make run-serve`, `make run-ingest` for quick dev loops.
+
+
+## About the Jetstream and data model 
+
+Bluesky is an open social network running on [the AT Protocol](https://github.com/bluesky-social/pds?tab=readme-ov-file#what-is-at-protocol) that [operates a firehose](https://docs.bsky.app/docs/advanced-guides/firehose), an authenticated stream of events with all activity on the network. Anyone can consume/produce against the firehose [following the developer guidelines.](https://docs.bsky.app/docs/support/developer-guidelines). 
+
+One of the key features developers can do is [create custom feeds of content](https://docs.bsky.app/docs/starter-templates/custom-feeds) based on either simple heuristics like regex, or collecting data from the firehose for machine-learning style feeds including lookup with embedding models, activity aggregation, etc.  
+
+This repo started exploring the idea of creating a custom feed and publishing it to my own PDS [in Go](https://github.com/veekaybee/gitfeed/blob/main/publishXRPC.go). It since moved to consuming directly from Jetstream, a lighter (and less stable) implementation that doesn't include the full scope of history that the firehose does [for every entry in a user's PDS.](https://jazco.dev/2024/09/24/jetstream/.) The tradeoff is that we are now consuming untrusted elements; however, since this is from the bluesky social relay, it's fine in ways that it might not be for other, less official relays.   
+
+## Looking at events in the Jetstream
+
+You can check GitHub events streaming to jetstream with: 
+
+```sh
+websocat wss://jetstream2.us-east.bsky.network/subscribe\?wantedCollections=app.bsky.feed.post | grep "github" | jq .
+```
+
+## Data Model:
+
+AtProto has its own data model, defined using [schemas called "Lexicons"](https://atproto.com/guides/lexicon). For posts and actions, they look like this. 
+
+```json5
+{
+  "did": "did:plc:eygmaihciaxprqvxpfvl6flk",
+  "time_us": 1725911162329308,
+  "kind": "commit",
+  "commit": {
+    "rev": "3l3qo2vutsw2b",
+    "operation": "create",
+    "collection": "app.bsky.feed.like",
+    "rkey": "3l3qo2vuowo2b",
+    "record": {
+      "$type": "app.bsky.feed.like",
+      "createdAt": "2024-09-09T19:46:02.102Z",
+      "subject": {
+        "cid": "bafyreidc6sydkkbchcyg62v77wbhzvb2mvytlmsychqgwf2xojjtirmzj4",
+        "uri": "at://did:plc:wa7b35aakoll7hugkrjtf3xf/app.bsky.feed.post/3l3pte3p2e325"
+      }
+    },
+    "cid": "bafyreidwaivazkwu67xztlmuobx35hs2lnfh3kolmgfmucldvhd3sgzcqi"
+  }
+}
+```
+
+DID is the ID of the PDS (user repository) where the action happened, the record type of "app.bsky.feed.post" is what we care about, and each record has both a text entry, which truncates the text, [and a facet](https://docs.bsky.app/docs/advanced-guides/post-richtext), which has all the contained links and rich text elements in the post. 
+
+Here's a full example of a GitHub link post:
+
+```sh
+{
+  "did": "did:plc:",
+  "time_us": 1732988544395778,
+  "type": "com",
+  "kind": "commit",
+  "commit": {
+    "rev": "",
+    "type": "c",
+    "operation": "create",
+    "collection": "app.bsky.feed.post",
+    "rkey": "",
+    "record": {
+      "$type": "app.bsky.feed.post",
+      "createdAt": "2024-11-29T17:42:14.541Z",
+      "embed": {
+        "$type": "app.bsky.embed.external",
+        "external": {
+          "description": "",
+          "thumb": {
+            "$type": "blob",
+            "ref": {
+              "$link": ""
+            },
+            "mimeType": "image/jpeg",
+            "size": 
+          },
+          "title": "",
+          "uri": ""
+        }
+      },
+      "facets": [
+        {
+          "features": [
+            {
+              "$type": "app.bsky.richtext.facet#link",
+              "uri": ""
+            }
+          ],
+          "index": {
+            "byteEnd": 85,
+            "byteStart": 54
+          }
+        }
+      ],
+      "langs": [
+        "en"
+      ],
+      "text": "..."
+    },
+    "cid": ""
+  }
+}
+```
+
+## Thanks
+
+Thanks to: AtProto devs, the [Bluesky docs](https://docs.bsky.app/) and everyone in the [API Touchers Discord](discord.gg/FS9U8A7F) who helped by patiently answering questions about the data model.