Initial commit
This commit is contained in:
primal
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
~/apps/.launch.sh "$@"
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
FROM golang:latest AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy shared module first
|
||||
COPY commons/ ./commons/
|
||||
|
||||
# Copy publisher module
|
||||
COPY publisher/ ./publisher/
|
||||
|
||||
# Build the binary
|
||||
WORKDIR /build/publisher
|
||||
RUN go mod download
|
||||
RUN CGO_ENABLED=0 go build -o publisher .
|
||||
|
||||
# Runtime stage
|
||||
FROM ubuntu:latest
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install runtime dependencies
|
||||
RUN apt-get update && apt-get install -y ca-certificates tzdata curl wget && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy binary from builder
|
||||
COPY --from=builder /build/publisher/publisher .
|
||||
|
||||
EXPOSE 4322
|
||||
|
||||
CMD ["./publisher"]
|
||||
@@ -0,0 +1,49 @@
|
||||
services:
|
||||
publisher:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: publisher/Dockerfile
|
||||
image: atproto-1440news-publisher
|
||||
container_name: atproto-1440news-publisher
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
env_file:
|
||||
- pds.env
|
||||
environment:
|
||||
DB_HOST: infra-postgres
|
||||
DB_PORT: 5432
|
||||
DB_USER: dba_1440_news
|
||||
DB_PASSWORD_FILE: /run/secrets/db_password
|
||||
DB_NAME: db_1440_news
|
||||
secrets:
|
||||
- db_password
|
||||
networks:
|
||||
- proxy
|
||||
- atproto
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
# Production: HTTPS with Let's Encrypt for api.1440.news
|
||||
- "traefik.http.routers.publisher-1440.rule=Host(`api.1440.news`)"
|
||||
- "traefik.http.routers.publisher-1440.entrypoints=https"
|
||||
- "traefik.http.routers.publisher-1440.tls.certresolver=letsencrypt-dns"
|
||||
# Production: HTTP to HTTPS redirect
|
||||
- "traefik.http.routers.publisher-1440-redirect.rule=Host(`api.1440.news`)"
|
||||
- "traefik.http.routers.publisher-1440-redirect.entrypoints=http"
|
||||
- "traefik.http.routers.publisher-1440-redirect.middlewares=https-redirect"
|
||||
- "traefik.http.middlewares.https-redirect.redirectscheme.scheme=https"
|
||||
- "traefik.http.middlewares.https-redirect.redirectscheme.permanent=true"
|
||||
# Local development
|
||||
- "traefik.http.routers.publisher-1440-local.rule=Host(`api.1440.localhost`)"
|
||||
- "traefik.http.routers.publisher-1440-local.entrypoints=http"
|
||||
# Shared service
|
||||
- "traefik.http.services.publisher-1440.loadbalancer.server.port=4322"
|
||||
|
||||
secrets:
|
||||
db_password:
|
||||
file: ../../../infra/postgres/secrets/dba_1440_news_password.txt
|
||||
|
||||
networks:
|
||||
proxy:
|
||||
external: true
|
||||
atproto:
|
||||
external: true
|
||||
@@ -0,0 +1,26 @@
|
||||
module github.com/1440news/publisher
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/1440news/commons v0.0.0
|
||||
github.com/jackc/pgx/v5 v5.7.5
|
||||
go.deanishe.net/favicon v0.1.0
|
||||
golang.org/x/image v0.26.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.6.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.1.0 // indirect
|
||||
github.com/friendsofgo/errors v0.9.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
golang.org/x/crypto v0.47.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
|
||||
)
|
||||
|
||||
replace github.com/1440news/commons => ../commons
|
||||
@@ -0,0 +1,144 @@
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94=
|
||||
github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk=
|
||||
github.com/friendsofgo/errors v0.9.2/go.mod h1:yCvFW5AkDIL9qn7suHVLiI/gH228n7PC4Pn44IGoTOI=
|
||||
github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
|
||||
github.com/go-openapi/strfmt v0.19.8/go.mod h1:qBBipho+3EoIqn6YDI+4RnQEtj6jT/IdKm+PAlXxSUc=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
|
||||
github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
|
||||
github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
|
||||
github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
|
||||
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
|
||||
github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
|
||||
github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
|
||||
github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
|
||||
github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
|
||||
github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
|
||||
github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
|
||||
github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
|
||||
github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
|
||||
github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
|
||||
github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
|
||||
github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
|
||||
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
|
||||
github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
|
||||
github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
|
||||
github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
|
||||
github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
|
||||
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
|
||||
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
|
||||
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
|
||||
github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
|
||||
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
|
||||
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
|
||||
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
|
||||
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
go.deanishe.net/favicon v0.1.0 h1:Afy941gjRik+DjUUcYHUxcztFEeFse2ITBkMMOlgefM=
|
||||
go.deanishe.net/favicon v0.1.0/go.mod h1:vIKVI+lUh8k3UAzaN4gjC+cpyatLQWmx0hVX4vLE8jU=
|
||||
go.mongodb.org/mongo-driver v1.4.2/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
|
||||
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
|
||||
golang.org/x/image v0.26.0 h1:4XjIFEZWQmCZi6Wv8BoxsDhRU3RVnLX04dToTDAEPlY=
|
||||
golang.org/x/image v0.26.0/go.mod h1:lcxbMFAovzpnJxzXS3nyL83K27tmqtKzIJpctK8YO5c=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.deanishe.net/favicon"
|
||||
"golang.org/x/image/draw"
|
||||
_ "golang.org/x/image/webp"
|
||||
)
|
||||
|
||||
// ImageUploadResult contains the uploaded blob and image dimensions
|
||||
type ImageUploadResult struct {
|
||||
Blob *BlobRef
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
// FetchFavicon tries to get a favicon URL for a site
|
||||
func (p *Publisher) FetchFavicon(siteURL string) string {
|
||||
if siteURL == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
if !strings.Contains(siteURL, "://") {
|
||||
siteURL = "https://" + siteURL
|
||||
}
|
||||
u, err := url.Parse(siteURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
finder := favicon.New(
|
||||
favicon.WithClient(p.httpClient),
|
||||
)
|
||||
|
||||
icons, err := finder.Find(siteURL)
|
||||
if err == nil && len(icons) > 0 {
|
||||
var bestIcon string
|
||||
var bestScore int
|
||||
|
||||
for _, icon := range icons {
|
||||
if icon.Width > 0 && icon.Width < 32 {
|
||||
continue
|
||||
}
|
||||
|
||||
lowerURL := strings.ToLower(icon.URL)
|
||||
if strings.Contains(lowerURL, "og-image") || strings.Contains(lowerURL, "og_image") ||
|
||||
strings.Contains(lowerURL, "opengraph") || strings.Contains(lowerURL, "twitter") {
|
||||
continue
|
||||
}
|
||||
|
||||
if icon.Width > 0 && icon.Height > 0 {
|
||||
ratio := float64(icon.Width) / float64(icon.Height)
|
||||
if ratio > 1.5 || ratio < 0.67 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
score := 0
|
||||
|
||||
if strings.Contains(lowerURL, "favicon") || strings.Contains(lowerURL, "icon") ||
|
||||
strings.Contains(lowerURL, "apple-touch") {
|
||||
score += 100
|
||||
}
|
||||
|
||||
if icon.MimeType == "image/png" {
|
||||
score += 50
|
||||
} else if icon.MimeType == "image/x-icon" || strings.HasSuffix(lowerURL, ".ico") {
|
||||
score += 40
|
||||
} else if icon.MimeType == "image/jpeg" {
|
||||
score += 10
|
||||
}
|
||||
|
||||
if icon.Width >= 64 && icon.Width <= 512 {
|
||||
score += 30
|
||||
} else if icon.Width > 0 {
|
||||
score += 10
|
||||
}
|
||||
|
||||
if score > bestScore {
|
||||
bestScore = score
|
||||
bestIcon = icon.URL
|
||||
}
|
||||
}
|
||||
|
||||
if bestIcon != "" {
|
||||
return bestIcon
|
||||
}
|
||||
|
||||
for _, icon := range icons {
|
||||
lowerURL := strings.ToLower(icon.URL)
|
||||
if !strings.Contains(lowerURL, "og-image") && !strings.Contains(lowerURL, "og_image") {
|
||||
return icon.URL
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host)
|
||||
}
|
||||
|
||||
func (p *Publisher) fetchAndUploadImage(session *PDSSession, imageURL string) *BlobRef {
|
||||
result := p.fetchAndUploadImageWithDimensions(session, imageURL)
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return result.Blob
|
||||
}
|
||||
|
||||
func upgradeImageURL(imageURL string) string {
|
||||
if strings.Contains(imageURL, "ichef.bbci.co.uk") {
|
||||
imageURL = strings.Replace(imageURL, "/standard/240/", "/standard/800/", 1)
|
||||
imageURL = strings.Replace(imageURL, "/standard/480/", "/standard/800/", 1)
|
||||
}
|
||||
return imageURL
|
||||
}
|
||||
|
||||
func (p *Publisher) fetchAndUploadImageWithDimensions(session *PDSSession, imageURL string) *ImageUploadResult {
|
||||
imageURL = upgradeImageURL(imageURL)
|
||||
|
||||
resp, err := p.httpClient.Get(imageURL)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
if strings.HasSuffix(strings.ToLower(imageURL), ".png") {
|
||||
contentType = "image/png"
|
||||
} else if strings.HasSuffix(strings.ToLower(imageURL), ".gif") {
|
||||
contentType = "image/gif"
|
||||
} else if strings.HasSuffix(strings.ToLower(imageURL), ".webp") {
|
||||
contentType = "image/webp"
|
||||
} else {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(contentType, "image/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
||||
if err != nil || len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
imgConfig, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
width, height := 1, 1
|
||||
if err == nil {
|
||||
width, height = imgConfig.Width, imgConfig.Height
|
||||
}
|
||||
|
||||
const maxBlobSize = 900 * 1024
|
||||
|
||||
if len(data) > maxBlobSize {
|
||||
img, _, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
scaleFactor := 0.9
|
||||
|
||||
for attempt := 0; attempt < 5; attempt++ {
|
||||
newWidth := int(float64(width) * scaleFactor)
|
||||
newHeight := int(float64(height) * scaleFactor)
|
||||
|
||||
if newWidth < 100 {
|
||||
newWidth = 100
|
||||
}
|
||||
if newHeight < 100 {
|
||||
newHeight = 100
|
||||
}
|
||||
|
||||
resized := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight))
|
||||
draw.CatmullRom.Scale(resized, resized.Bounds(), img, img.Bounds(), draw.Over, nil)
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, resized, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if buf.Len() <= maxBlobSize {
|
||||
data = buf.Bytes()
|
||||
width = newWidth
|
||||
height = newHeight
|
||||
contentType = "image/jpeg"
|
||||
break
|
||||
}
|
||||
|
||||
scaleFactor *= 0.8
|
||||
}
|
||||
|
||||
if len(data) > maxBlobSize {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
blob, err := p.UploadBlob(session, data, contentType)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &ImageUploadResult{
|
||||
Blob: blob,
|
||||
Width: width,
|
||||
Height: height,
|
||||
}
|
||||
}
|
||||
|
||||
// FetchFaviconBytes downloads a favicon/icon from a URL
|
||||
func FetchFaviconBytes(siteURL string) ([]byte, string, error) {
|
||||
if !strings.HasPrefix(siteURL, "http") {
|
||||
siteURL = "https://" + siteURL
|
||||
}
|
||||
|
||||
u, err := url.Parse(siteURL)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
|
||||
finder := favicon.New(
|
||||
favicon.WithClient(client),
|
||||
favicon.IgnoreNoSize,
|
||||
)
|
||||
|
||||
icons, err := finder.Find(siteURL)
|
||||
if err != nil || len(icons) == 0 {
|
||||
googleURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host)
|
||||
return fetchIconBytes(client, googleURL)
|
||||
}
|
||||
|
||||
var iconURLs []string
|
||||
for _, icon := range icons {
|
||||
if icon.Width > 0 && icon.Width < 32 {
|
||||
continue
|
||||
}
|
||||
if icon.MimeType == "image/png" || icon.MimeType == "image/jpeg" {
|
||||
iconURLs = append([]string{icon.URL}, iconURLs...)
|
||||
} else {
|
||||
iconURLs = append(iconURLs, icon.URL)
|
||||
}
|
||||
}
|
||||
|
||||
if len(iconURLs) == 0 {
|
||||
for _, icon := range icons {
|
||||
iconURLs = append(iconURLs, icon.URL)
|
||||
}
|
||||
}
|
||||
|
||||
for _, iconURL := range iconURLs {
|
||||
data, mimeType, err := fetchIconBytes(client, iconURL)
|
||||
if err == nil && len(data) > 0 {
|
||||
return data, mimeType, nil
|
||||
}
|
||||
}
|
||||
|
||||
googleURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host)
|
||||
return fetchIconBytes(client, googleURL)
|
||||
}
|
||||
|
||||
func fetchIconBytes(client *http.Client, iconURL string) ([]byte, string, error) {
|
||||
resp, err := client.Get(iconURL)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
if strings.HasSuffix(iconURL, ".png") {
|
||||
contentType = "image/png"
|
||||
} else if strings.HasSuffix(iconURL, ".ico") {
|
||||
contentType = "image/x-icon"
|
||||
} else {
|
||||
contentType = "image/png"
|
||||
}
|
||||
}
|
||||
|
||||
return data, contentType, nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/1440news/commons"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// scanItems scans rows into Item structs
|
||||
func scanItems(rows pgx.Rows) ([]*shared.Item, error) {
|
||||
var items []*shared.Item
|
||||
|
||||
for rows.Next() {
|
||||
item := &shared.Item{}
|
||||
var guid, title, link, description, content, author *string
|
||||
var pubDate, updatedAt, publishedAt *interface{}
|
||||
var publishedUri *string
|
||||
var enclosureUrl, enclosureType *string
|
||||
var enclosureLength *int64
|
||||
var imageUrlsJSON, tagsJSON *string
|
||||
|
||||
err := rows.Scan(
|
||||
&item.FeedURL, &guid, &title, &link,
|
||||
&description, &content, &author, &pubDate,
|
||||
&item.DiscoveredAt, &updatedAt,
|
||||
&enclosureUrl, &enclosureType, &enclosureLength,
|
||||
&imageUrlsJSON, &tagsJSON,
|
||||
&publishedAt, &publishedUri,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
item.GUID = shared.StringValue(guid)
|
||||
item.Title = shared.StringValue(title)
|
||||
item.Link = shared.StringValue(link)
|
||||
item.Description = shared.StringValue(description)
|
||||
item.Content = shared.StringValue(content)
|
||||
item.Author = shared.StringValue(author)
|
||||
item.PublishedUri = shared.StringValue(publishedUri)
|
||||
|
||||
if pubDate != nil {
|
||||
if t, ok := (*pubDate).(interface{ Time() interface{} }); ok {
|
||||
if tm, ok := t.Time().(interface{ IsZero() bool }); ok && !tm.IsZero() {
|
||||
// Handle time conversion
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if enclosureUrl != nil && *enclosureUrl != "" {
|
||||
item.Enclosure = &shared.Enclosure{
|
||||
URL: *enclosureUrl,
|
||||
Type: shared.StringValue(enclosureType),
|
||||
}
|
||||
if enclosureLength != nil {
|
||||
item.Enclosure.Length = *enclosureLength
|
||||
}
|
||||
}
|
||||
|
||||
if imageUrlsJSON != nil && *imageUrlsJSON != "" {
|
||||
json.Unmarshal([]byte(*imageUrlsJSON), &item.ImageURLs)
|
||||
}
|
||||
|
||||
if tagsJSON != nil && *tagsJSON != "" {
|
||||
json.Unmarshal([]byte(*tagsJSON), &item.Tags)
|
||||
}
|
||||
|
||||
items = append(items, item)
|
||||
}
|
||||
|
||||
return items, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/1440news/commons"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("Starting publisher service...")
|
||||
|
||||
// Open database connection
|
||||
db, err := shared.OpenDatabase("")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Load PDS configuration
|
||||
pdsHost := os.Getenv("PDS_HOST")
|
||||
if pdsHost == "" {
|
||||
pdsHost = "https://pds.1440.news"
|
||||
}
|
||||
|
||||
feedPassword := os.Getenv("FEED_PASSWORD")
|
||||
if feedPassword == "" {
|
||||
feedPassword = "feed1440!"
|
||||
}
|
||||
|
||||
// Create publisher service
|
||||
pub := NewPublisherService(db, pdsHost, feedPassword)
|
||||
|
||||
// Start HTTP server
|
||||
go func() {
|
||||
if err := pub.StartServer("0.0.0.0:4322"); err != nil {
|
||||
log.Fatalf("Server error: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Start publish loop
|
||||
go pub.StartPublishLoop()
|
||||
|
||||
fmt.Printf("Publisher service running (PDS: %s)\n", pdsHost)
|
||||
|
||||
// Wait for shutdown signal
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-sigChan
|
||||
|
||||
fmt.Println("Shutting down publisher service...")
|
||||
}
|
||||
|
||||
// loadEnvFile loads environment variables from a file (e.g., pds.env)
|
||||
func loadEnvFile(filename string) {
|
||||
data, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return // File doesn't exist or can't be read
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
parts := strings.SplitN(line, "=", 2)
|
||||
if len(parts) == 2 {
|
||||
key := strings.TrimSpace(parts[0])
|
||||
value := strings.TrimSpace(parts[1])
|
||||
if os.Getenv(key) == "" {
|
||||
os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PublisherService manages publishing items to AT Protocol PDS
|
||||
type PublisherService struct {
|
||||
db *shared.DB
|
||||
publisher *Publisher
|
||||
pdsHost string
|
||||
feedPassword string
|
||||
}
|
||||
|
||||
// NewPublisherService creates a new publisher service
|
||||
func NewPublisherService(db *shared.DB, pdsHost, feedPassword string) *PublisherService {
|
||||
return &PublisherService{
|
||||
db: db,
|
||||
publisher: NewPublisher(pdsHost),
|
||||
pdsHost: pdsHost,
|
||||
feedPassword: feedPassword,
|
||||
}
|
||||
}
|
||||
|
||||
// StartPublishLoop runs the automatic publishing loop
|
||||
func (s *PublisherService) StartPublishLoop() {
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Run immediately on start
|
||||
s.publishPendingItems()
|
||||
|
||||
for range ticker.C {
|
||||
s.publishPendingItems()
|
||||
}
|
||||
}
|
||||
|
||||
// publishPendingItems publishes unpublished items for all enabled feeds
|
||||
func (s *PublisherService) publishPendingItems() {
|
||||
// Get all feeds with publish_status = 'pass'
|
||||
rows, err := s.db.Query(`
|
||||
SELECT url, publish_account
|
||||
FROM feeds
|
||||
WHERE publish_status = 'pass' AND publish_account IS NOT NULL
|
||||
`)
|
||||
if err != nil {
|
||||
fmt.Printf("Publish loop: query error: %v\n", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var feeds []struct {
|
||||
URL string
|
||||
Account string
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var url, account string
|
||||
if err := rows.Scan(&url, &account); err != nil {
|
||||
continue
|
||||
}
|
||||
feeds = append(feeds, struct {
|
||||
URL string
|
||||
Account string
|
||||
}{url, account})
|
||||
}
|
||||
|
||||
if len(feeds) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("Publish loop: checking %d feeds\n", len(feeds))
|
||||
|
||||
for _, feed := range feeds {
|
||||
s.publishFeedItems(feed.URL, feed.Account)
|
||||
}
|
||||
}
|
||||
|
||||
// publishFeedItems publishes unpublished items for a single feed
|
||||
func (s *PublisherService) publishFeedItems(feedURL, account string) {
|
||||
// Get unpublished items (limit to 5 per cycle to avoid overwhelming)
|
||||
items, err := s.GetUnpublishedItems(feedURL, 5)
|
||||
if err != nil || len(items) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Authenticate with the feed account
|
||||
session, err := s.publisher.CreateSession(account, s.feedPassword)
|
||||
if err != nil {
|
||||
fmt.Printf("Publish: auth failed for %s: %v\n", account, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
uri, err := s.publisher.PublishItem(session, item)
|
||||
if err != nil {
|
||||
fmt.Printf("Publish: failed to publish %s: %v\n", item.GUID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.MarkItemPublished(item.FeedURL, item.GUID, uri); err != nil {
|
||||
fmt.Printf("Publish: failed to mark published %s: %v\n", item.GUID, err)
|
||||
}
|
||||
|
||||
// Small delay between posts
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// GetUnpublishedItems returns unpublished items for a feed
|
||||
func (s *PublisherService) GetUnpublishedItems(feedURL string, limit int) ([]*shared.Item, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT feed_url, guid, title, link, description, content, author, pub_date, discovered_at, updated_at,
|
||||
enclosure_url, enclosure_type, enclosure_length, image_urls, tags,
|
||||
published_at, published_uri
|
||||
FROM items
|
||||
WHERE feed_url = $1 AND published_at IS NULL
|
||||
ORDER BY pub_date ASC
|
||||
LIMIT $2
|
||||
`, feedURL, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanItems(rows)
|
||||
}
|
||||
|
||||
// MarkItemPublished marks an item as published with the given URI
|
||||
func (s *PublisherService) MarkItemPublished(feedURL, guid, uri string) error {
|
||||
_, err := s.db.Exec(`
|
||||
UPDATE items SET published_at = NOW(), published_uri = $1 WHERE feed_url = $2 AND guid = $3
|
||||
`, uri, feedURL, guid)
|
||||
return err
|
||||
}
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CreateSession authenticates with the PDS and returns a session
|
||||
func (p *Publisher) CreateSession(handle, password string) (*PDSSession, error) {
|
||||
payload := map[string]string{
|
||||
"identifier": handle,
|
||||
"password": password,
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := p.httpClient.Post(
|
||||
p.pdsHost+"/xrpc/com.atproto.server.createSession",
|
||||
"application/json",
|
||||
bytes.NewReader(body),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("auth failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
var session PDSSession
|
||||
if err := json.NewDecoder(resp.Body).Decode(&session); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &session, nil
|
||||
}
|
||||
|
||||
// CreateAccount creates a new account on the PDS
|
||||
func (p *Publisher) CreateAccount(handle, email, password, inviteCode string) (*PDSSession, error) {
|
||||
payload := map[string]interface{}{
|
||||
"handle": handle,
|
||||
"email": email,
|
||||
"password": password,
|
||||
}
|
||||
if inviteCode != "" {
|
||||
payload["inviteCode"] = inviteCode
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := p.httpClient.Post(
|
||||
p.pdsHost+"/xrpc/com.atproto.server.createAccount",
|
||||
"application/json",
|
||||
bytes.NewReader(body),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("create account failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
var session PDSSession
|
||||
if err := json.Unmarshal(respBody, &session); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &session, nil
|
||||
}
|
||||
|
||||
// CreateInviteCode creates an invite code using PDS admin password (Basic Auth)
|
||||
func (p *Publisher) CreateInviteCode(adminPassword string, useCount int) (string, error) {
|
||||
payload := map[string]interface{}{
|
||||
"useCount": useCount,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.server.createInviteCode", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.SetBasicAuth("admin", adminPassword)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("create invite failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Code string `json:"code"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return result.Code, nil
|
||||
}
|
||||
|
||||
// FollowAccount creates a follow record from the authenticated session to the target DID
|
||||
func (p *Publisher) FollowAccount(session *PDSSession, targetDID string) error {
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
record := map[string]interface{}{
|
||||
"$type": "app.bsky.graph.follow",
|
||||
"subject": targetDID,
|
||||
"createdAt": now,
|
||||
}
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"repo": session.DID,
|
||||
"collection": "app.bsky.graph.follow",
|
||||
"record": record,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.createRecord", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+session.AccessJwt)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("follow failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FollowAsDirectory logs in as the directory account and follows the target DID
|
||||
func (p *Publisher) FollowAsDirectory(targetDID string) error {
|
||||
dirHandle := os.Getenv("DIRECTORY_HANDLE")
|
||||
dirPassword := os.Getenv("DIRECTORY_PASSWORD")
|
||||
|
||||
if dirHandle == "" || dirPassword == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
session, err := p.CreateSession(dirHandle, dirPassword)
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory login failed: %w", err)
|
||||
}
|
||||
|
||||
return p.FollowAccount(session, targetDID)
|
||||
}
|
||||
+342
@@ -0,0 +1,342 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BlobRef represents a blob reference for profile images
|
||||
type BlobRef struct {
|
||||
Type string `json:"$type"`
|
||||
Ref Link `json:"ref"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type Link struct {
|
||||
Link string `json:"$link"`
|
||||
}
|
||||
|
||||
// UploadBlob uploads an image to the PDS and returns a blob reference
|
||||
func (p *Publisher) UploadBlob(session *PDSSession, data []byte, mimeType string) (*BlobRef, error) {
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.uploadBlob", bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", mimeType)
|
||||
req.Header.Set("Authorization", "Bearer "+session.AccessJwt)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("upload blob failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Blob BlobRef `json:"blob"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result.Blob, nil
|
||||
}
|
||||
|
||||
// UpdateProfile updates the profile for an account
|
||||
func (p *Publisher) UpdateProfile(session *PDSSession, displayName, description string, avatar *BlobRef) error {
|
||||
getReq, err := http.NewRequest("GET",
|
||||
p.pdsHost+"/xrpc/com.atproto.repo.getRecord?repo="+session.DID+"&collection=app.bsky.actor.profile&rkey=self",
|
||||
nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
getReq.Header.Set("Authorization", "Bearer "+session.AccessJwt)
|
||||
|
||||
getResp, err := p.httpClient.Do(getReq)
|
||||
|
||||
var existingCID string
|
||||
profile := map[string]interface{}{
|
||||
"$type": "app.bsky.actor.profile",
|
||||
}
|
||||
|
||||
if err == nil && getResp.StatusCode == http.StatusOK {
|
||||
defer getResp.Body.Close()
|
||||
var existing struct {
|
||||
CID string `json:"cid"`
|
||||
Value map[string]interface{} `json:"value"`
|
||||
}
|
||||
if json.NewDecoder(getResp.Body).Decode(&existing) == nil {
|
||||
existingCID = existing.CID
|
||||
profile = existing.Value
|
||||
}
|
||||
} else if getResp != nil {
|
||||
getResp.Body.Close()
|
||||
}
|
||||
|
||||
if displayName != "" {
|
||||
profile["displayName"] = displayName
|
||||
}
|
||||
if description != "" {
|
||||
profile["description"] = description
|
||||
}
|
||||
if avatar != nil {
|
||||
profile["avatar"] = avatar
|
||||
}
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"repo": session.DID,
|
||||
"collection": "app.bsky.actor.profile",
|
||||
"rkey": "self",
|
||||
"record": profile,
|
||||
}
|
||||
if existingCID != "" {
|
||||
payload["swapRecord"] = existingCID
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.putRecord", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+session.AccessJwt)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("update profile failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAllPosts deletes all posts from an account
|
||||
func (p *Publisher) DeleteAllPosts(session *PDSSession) (int, error) {
|
||||
deleted := 0
|
||||
cursor := ""
|
||||
|
||||
for {
|
||||
listURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=app.bsky.feed.post&limit=100",
|
||||
p.pdsHost, session.DID)
|
||||
if cursor != "" {
|
||||
listURL += "&cursor=" + url.QueryEscape(cursor)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", listURL, nil)
|
||||
if err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+session.AccessJwt)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Records []struct {
|
||||
URI string `json:"uri"`
|
||||
} `json:"records"`
|
||||
Cursor string `json:"cursor"`
|
||||
}
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return deleted, fmt.Errorf("list records failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
if len(result.Records) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, record := range result.Records {
|
||||
parts := strings.Split(record.URI, "/")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
rkey := parts[len(parts)-1]
|
||||
|
||||
if err := p.DeleteRecord(session, "app.bsky.feed.post", rkey); err != nil {
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
cursor = result.Cursor
|
||||
if cursor == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// DeleteRecord deletes a single record from an account
|
||||
func (p *Publisher) DeleteRecord(session *PDSSession, collection, rkey string) error {
|
||||
payload := map[string]interface{}{
|
||||
"repo": session.DID,
|
||||
"collection": collection,
|
||||
"rkey": rkey,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.deleteRecord", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+session.AccessJwt)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("delete record failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAccount deletes an account using PDS admin API
|
||||
func (p *Publisher) DeleteAccount(adminPassword, did string) error {
|
||||
payload := map[string]interface{}{
|
||||
"did": did,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.admin.deleteAccount", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.SetBasicAuth("admin", adminPassword)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("delete account failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TakedownAccount applies a takedown to an account
|
||||
func (p *Publisher) TakedownAccount(adminPassword, did, reason string) error {
|
||||
payload := map[string]interface{}{
|
||||
"subject": map[string]interface{}{
|
||||
"$type": "com.atproto.admin.defs#repoRef",
|
||||
"did": did,
|
||||
},
|
||||
"takedown": map[string]interface{}{
|
||||
"applied": true,
|
||||
"ref": reason,
|
||||
},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.admin.updateSubjectStatus", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.SetBasicAuth("admin", adminPassword)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("takedown account failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreAccount removes a takedown from an account
|
||||
func (p *Publisher) RestoreAccount(adminPassword, did string) error {
|
||||
payload := map[string]interface{}{
|
||||
"subject": map[string]interface{}{
|
||||
"$type": "com.atproto.admin.defs#repoRef",
|
||||
"did": did,
|
||||
},
|
||||
"takedown": map[string]interface{}{
|
||||
"applied": false,
|
||||
},
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.admin.updateSubjectStatus", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.SetBasicAuth("admin", adminPassword)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("restore account failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+395
@@ -0,0 +1,395 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/1440news/commons"
|
||||
)
|
||||
|
||||
// Publisher handles posting items to AT Protocol PDS
|
||||
type Publisher struct {
|
||||
pdsHost string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// PDSSession holds authentication info for a PDS account
|
||||
type PDSSession struct {
|
||||
DID string `json:"did"`
|
||||
Handle string `json:"handle"`
|
||||
AccessJwt string `json:"accessJwt"`
|
||||
RefreshJwt string `json:"refreshJwt"`
|
||||
}
|
||||
|
||||
// BskyPost represents an app.bsky.feed.post record
|
||||
type BskyPost struct {
|
||||
Type string `json:"$type"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
Facets []BskyFacet `json:"facets,omitempty"`
|
||||
Embed *BskyEmbed `json:"embed,omitempty"`
|
||||
}
|
||||
|
||||
type BskyFacet struct {
|
||||
Index BskyByteSlice `json:"index"`
|
||||
Features []BskyFeature `json:"features"`
|
||||
}
|
||||
|
||||
type BskyByteSlice struct {
|
||||
ByteStart int `json:"byteStart"`
|
||||
ByteEnd int `json:"byteEnd"`
|
||||
}
|
||||
|
||||
type BskyFeature struct {
|
||||
Type string `json:"$type"`
|
||||
URI string `json:"uri,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
}
|
||||
|
||||
type BskyEmbed struct {
|
||||
Type string `json:"$type"`
|
||||
External *BskyExternal `json:"external,omitempty"`
|
||||
Images []BskyImage `json:"images,omitempty"`
|
||||
}
|
||||
|
||||
type BskyExternal struct {
|
||||
URI string `json:"uri"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Thumb *BlobRef `json:"thumb,omitempty"`
|
||||
}
|
||||
|
||||
type BskyImage struct {
|
||||
Alt string `json:"alt"`
|
||||
Image *BlobRef `json:"image"`
|
||||
AspectRatio *BskyAspectRatio `json:"aspectRatio,omitempty"`
|
||||
}
|
||||
|
||||
type BskyAspectRatio struct {
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
// NewPublisher creates a new Publisher instance
|
||||
func NewPublisher(pdsHost string) *Publisher {
|
||||
return &Publisher{
|
||||
pdsHost: pdsHost,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TID alphabet for base32-sortable encoding
|
||||
const tidAlphabet = "234567abcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
// GenerateRkey creates a deterministic TID-format rkey from a GUID and timestamp
|
||||
func GenerateRkey(guid string, timestamp time.Time) string {
|
||||
if guid == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
microsInt := timestamp.UnixMicro()
|
||||
if microsInt < 0 {
|
||||
microsInt = 0
|
||||
}
|
||||
micros := uint64(microsInt) & ((1 << 53) - 1)
|
||||
|
||||
hash := sha256.Sum256([]byte(guid))
|
||||
h1 := uint64(hash[0]) ^ uint64(hash[2]) ^ uint64(hash[4]) ^ uint64(hash[6])
|
||||
h2 := uint64(hash[1]) ^ uint64(hash[3]) ^ uint64(hash[5]) ^ uint64(hash[7])
|
||||
clockID := (h1 << 2) | (h2 >> 6)
|
||||
clockID = clockID & ((1 << 10) - 1)
|
||||
|
||||
tid := (micros << 10) | clockID
|
||||
|
||||
var result [13]byte
|
||||
for i := 12; i >= 0; i-- {
|
||||
result[i] = tidAlphabet[tid&0x1f]
|
||||
tid >>= 5
|
||||
}
|
||||
|
||||
return string(result[:])
|
||||
}
|
||||
|
||||
// extractURLs finds all URLs in a string
|
||||
func extractURLs(text string) []string {
|
||||
urlRegex := regexp.MustCompile(`https?://[^\s<>"'\)]+`)
|
||||
matches := urlRegex.FindAllString(text, -1)
|
||||
|
||||
var urls []string
|
||||
for _, u := range matches {
|
||||
u = strings.TrimRight(u, ".,;:!?")
|
||||
if u != "" {
|
||||
urls = append(urls, u)
|
||||
}
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// toCamelCaseTag converts a tag string to camelCase hashtag format
|
||||
func toCamelCaseTag(tag string) string {
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
tag = strings.TrimPrefix(tag, "#")
|
||||
|
||||
words := strings.FieldsFunc(tag, func(r rune) bool {
|
||||
return r == ' ' || r == '-' || r == '_'
|
||||
})
|
||||
|
||||
if len(words) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
if len(words) == 1 {
|
||||
return strings.ToLower(words[0])
|
||||
}
|
||||
|
||||
var result strings.Builder
|
||||
for i, word := range words {
|
||||
if word == "" {
|
||||
continue
|
||||
}
|
||||
runes := []rune(word)
|
||||
if len(runes) > 0 {
|
||||
if i == 0 || result.Len() == 0 {
|
||||
result.WriteString(strings.ToLower(word))
|
||||
} else {
|
||||
result.WriteString(strings.ToUpper(string(runes[0])))
|
||||
if len(runes) > 1 {
|
||||
result.WriteString(strings.ToLower(string(runes[1:])))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
|
||||
// formatTagsForPost converts item tags to hashtag text and facets
|
||||
func formatTagsForPost(tags []string, textOffset int) (string, []BskyFacet) {
|
||||
if len(tags) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
seen := make(map[string]bool)
|
||||
var hashtags []string
|
||||
for _, tag := range tags {
|
||||
camel := toCamelCaseTag(tag)
|
||||
if camel == "" || seen[strings.ToLower(camel)] {
|
||||
continue
|
||||
}
|
||||
seen[strings.ToLower(camel)] = true
|
||||
hashtags = append(hashtags, camel)
|
||||
}
|
||||
|
||||
if len(hashtags) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if len(hashtags) > 5 {
|
||||
hashtags = hashtags[:5]
|
||||
}
|
||||
|
||||
var line strings.Builder
|
||||
var facets []BskyFacet
|
||||
currentOffset := textOffset
|
||||
|
||||
for i, ht := range hashtags {
|
||||
if i > 0 {
|
||||
line.WriteString(" ")
|
||||
currentOffset++
|
||||
}
|
||||
|
||||
hashtagText := "#" + ht
|
||||
byteStart := currentOffset
|
||||
byteEnd := currentOffset + len(hashtagText)
|
||||
|
||||
line.WriteString(hashtagText)
|
||||
|
||||
facets = append(facets, BskyFacet{
|
||||
Index: BskyByteSlice{
|
||||
ByteStart: byteStart,
|
||||
ByteEnd: byteEnd,
|
||||
},
|
||||
Features: []BskyFeature{{
|
||||
Type: "app.bsky.richtext.facet#tag",
|
||||
Tag: ht,
|
||||
}},
|
||||
})
|
||||
|
||||
currentOffset = byteEnd
|
||||
}
|
||||
|
||||
return line.String(), facets
|
||||
}
|
||||
|
||||
// PublishItem posts a feed item to the PDS
|
||||
func (p *Publisher) PublishItem(session *PDSSession, item *shared.Item) (string, error) {
|
||||
if item.GUID == "" && item.Link == "" {
|
||||
return "", fmt.Errorf("item has no GUID or link, cannot publish")
|
||||
}
|
||||
|
||||
urlSet := make(map[string]bool)
|
||||
var allURLs []string
|
||||
|
||||
if item.Link != "" {
|
||||
urlSet[item.Link] = true
|
||||
allURLs = append(allURLs, item.Link)
|
||||
}
|
||||
|
||||
descURLs := extractURLs(item.Description)
|
||||
for _, u := range descURLs {
|
||||
if strings.Contains(u, "news.ycombinator.com/item") && !urlSet[u] {
|
||||
urlSet[u] = true
|
||||
allURLs = append(allURLs, u)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(allURLs) < 2 && item.Enclosure != nil && item.Enclosure.URL != "" {
|
||||
encType := strings.ToLower(item.Enclosure.Type)
|
||||
if strings.HasPrefix(encType, "audio/") || strings.HasPrefix(encType, "video/") {
|
||||
if !urlSet[item.Enclosure.URL] {
|
||||
currentURLLen := 0
|
||||
for _, u := range allURLs {
|
||||
currentURLLen += len(u) + 2
|
||||
}
|
||||
enclosureLen := len(item.Enclosure.URL) + 2
|
||||
if currentURLLen+enclosureLen < 235 {
|
||||
urlSet[item.Enclosure.URL] = true
|
||||
allURLs = append(allURLs, item.Enclosure.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
primaryURL := ""
|
||||
if len(allURLs) > 0 {
|
||||
primaryURL = allURLs[0]
|
||||
}
|
||||
|
||||
createdAt := time.Now()
|
||||
if !item.PubDate.IsZero() {
|
||||
createdAt = item.PubDate
|
||||
}
|
||||
|
||||
postText := ""
|
||||
var facets []BskyFacet
|
||||
|
||||
if len(item.Tags) > 0 {
|
||||
tagLine, tagFacets := formatTagsForPost(item.Tags, 0)
|
||||
postText = tagLine
|
||||
facets = tagFacets
|
||||
}
|
||||
|
||||
post := BskyPost{
|
||||
Type: "app.bsky.feed.post",
|
||||
Text: postText,
|
||||
CreatedAt: createdAt.Format(time.RFC3339),
|
||||
Facets: facets,
|
||||
}
|
||||
|
||||
if primaryURL != "" {
|
||||
external := &BskyExternal{
|
||||
URI: primaryURL,
|
||||
Title: item.Title,
|
||||
Description: truncate(stripHTML(item.Description), 300),
|
||||
}
|
||||
|
||||
if len(item.ImageURLs) > 0 {
|
||||
if thumb := p.fetchAndUploadImage(session, item.ImageURLs[0]); thumb != nil {
|
||||
external.Thumb = thumb
|
||||
}
|
||||
}
|
||||
|
||||
post.Embed = &BskyEmbed{
|
||||
Type: "app.bsky.embed.external",
|
||||
External: external,
|
||||
}
|
||||
}
|
||||
|
||||
guidForRkey := item.GUID
|
||||
if guidForRkey == "" {
|
||||
guidForRkey = item.Link
|
||||
}
|
||||
rkeyTime := item.PubDate
|
||||
if rkeyTime.IsZero() {
|
||||
rkeyTime = item.DiscoveredAt
|
||||
}
|
||||
rkey := GenerateRkey(guidForRkey, rkeyTime)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"repo": session.DID,
|
||||
"collection": "app.bsky.feed.post",
|
||||
"rkey": rkey,
|
||||
"record": post,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.createRecord", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+session.AccessJwt)
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("create record failed: %s - %s", resp.Status, string(respBody))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
URI string `json:"uri"`
|
||||
CID string `json:"cid"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return result.URI, nil
|
||||
}
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen-3] + "..."
|
||||
}
|
||||
|
||||
func stripHTML(s string) string {
|
||||
tagRegex := regexp.MustCompile(`<[^>]*>`)
|
||||
s = tagRegex.ReplaceAllString(s, "")
|
||||
|
||||
s = strings.ReplaceAll(s, "&", "&")
|
||||
s = strings.ReplaceAll(s, "<", "<")
|
||||
s = strings.ReplaceAll(s, ">", ">")
|
||||
s = strings.ReplaceAll(s, """, "\"")
|
||||
s = strings.ReplaceAll(s, "'", "'")
|
||||
s = strings.ReplaceAll(s, " ", " ")
|
||||
|
||||
spaceRegex := regexp.MustCompile(`\s+`)
|
||||
s = spaceRegex.ReplaceAllString(s, " ")
|
||||
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
@@ -0,0 +1,726 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/1440news/commons"
|
||||
)
|
||||
|
||||
// StartServer starts the HTTP server for the publisher API
|
||||
func (s *PublisherService) StartServer(addr string) error {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// Health check
|
||||
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("OK"))
|
||||
})
|
||||
|
||||
// Publishing APIs
|
||||
mux.HandleFunc("/api/enablePublish", s.handleEnablePublish)
|
||||
mux.HandleFunc("/api/disablePublish", s.handleDisablePublish)
|
||||
mux.HandleFunc("/api/setPublishStatus", s.handleSetPublishStatus)
|
||||
mux.HandleFunc("/api/publishEnabled", s.handlePublishEnabled)
|
||||
mux.HandleFunc("/api/publishDenied", s.handlePublishDenied)
|
||||
mux.HandleFunc("/api/publishCandidates", s.handlePublishCandidates)
|
||||
mux.HandleFunc("/api/unpublishedItems", s.handleUnpublishedItems)
|
||||
mux.HandleFunc("/api/testPublish", s.handleTestPublish)
|
||||
mux.HandleFunc("/api/publishFeed", s.handlePublishFeed)
|
||||
mux.HandleFunc("/api/publishFeedFull", s.handlePublishFeedFull)
|
||||
mux.HandleFunc("/api/createAccount", s.handleCreateAccount)
|
||||
mux.HandleFunc("/api/updateProfile", s.handleUpdateProfile)
|
||||
mux.HandleFunc("/api/deriveHandle", s.handleDeriveHandle)
|
||||
mux.HandleFunc("/api/resetAllPublishing", s.handleResetAllPublishing)
|
||||
mux.HandleFunc("/api/refreshProfiles", s.handleRefreshProfiles)
|
||||
|
||||
fmt.Printf("Publisher API running at http://%s\n", addr)
|
||||
return http.ListenAndServe(addr, mux)
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleEnablePublish(w http.ResponseWriter, r *http.Request) {
|
||||
feedURL := r.URL.Query().Get("url")
|
||||
account := r.URL.Query().Get("account")
|
||||
if feedURL == "" {
|
||||
http.Error(w, "url parameter required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
feedURL = shared.NormalizeURL(feedURL)
|
||||
|
||||
if account == "" {
|
||||
account = shared.DeriveHandleFromFeed(feedURL)
|
||||
if account == "" {
|
||||
http.Error(w, "could not derive account handle from URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.SetPublishStatus(feedURL, "pass", account); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
count, _ := s.GetUnpublishedItemCount(feedURL)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "pass",
|
||||
"url": feedURL,
|
||||
"account": account,
|
||||
"unpublished_items": count,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleDisablePublish(w http.ResponseWriter, r *http.Request) {
|
||||
feedURL := r.URL.Query().Get("url")
|
||||
if feedURL == "" {
|
||||
http.Error(w, "url parameter required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
feedURL = shared.NormalizeURL(feedURL)
|
||||
|
||||
if err := s.SetPublishStatus(feedURL, "skip", ""); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "skip",
|
||||
"url": feedURL,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleSetPublishStatus(w http.ResponseWriter, r *http.Request) {
|
||||
feedURL := r.URL.Query().Get("url")
|
||||
status := r.URL.Query().Get("status")
|
||||
account := r.URL.Query().Get("account")
|
||||
|
||||
if feedURL == "" {
|
||||
http.Error(w, "url parameter required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if status != "pass" && status != "skip" && status != "hold" && status != "drop" {
|
||||
http.Error(w, "status must be 'pass', 'hold', 'skip', or 'drop'", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
feedURL = shared.NormalizeURL(feedURL)
|
||||
|
||||
result := map[string]interface{}{
|
||||
"url": feedURL,
|
||||
"status": status,
|
||||
}
|
||||
|
||||
if status == "pass" {
|
||||
if account == "" {
|
||||
account = shared.DeriveHandleFromFeed(feedURL)
|
||||
}
|
||||
result["account"] = account
|
||||
}
|
||||
|
||||
if err := s.SetPublishStatus(feedURL, status, account); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(result)
|
||||
}
|
||||
|
||||
func (s *PublisherService) handlePublishEnabled(w http.ResponseWriter, r *http.Request) {
|
||||
feeds, err := s.GetFeedsByPublishStatus("pass")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
type FeedPublishInfo struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Account string `json:"account"`
|
||||
UnpublishedCount int `json:"unpublished_count"`
|
||||
}
|
||||
|
||||
var result []FeedPublishInfo
|
||||
for _, f := range feeds {
|
||||
count, _ := s.GetUnpublishedItemCount(f.URL)
|
||||
result = append(result, FeedPublishInfo{
|
||||
URL: f.URL,
|
||||
Title: f.Title,
|
||||
Account: f.PublishAccount,
|
||||
UnpublishedCount: count,
|
||||
})
|
||||
}
|
||||
|
||||
if result == nil {
|
||||
result = []FeedPublishInfo{}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(result)
|
||||
}
|
||||
|
||||
func (s *PublisherService) handlePublishDenied(w http.ResponseWriter, r *http.Request) {
|
||||
feeds, err := s.GetFeedsByPublishStatus("skip")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
type FeedDeniedInfo struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
SourceHost string `json:"source_host"`
|
||||
}
|
||||
|
||||
var result []FeedDeniedInfo
|
||||
for _, f := range feeds {
|
||||
result = append(result, FeedDeniedInfo{
|
||||
URL: f.URL,
|
||||
Title: f.Title,
|
||||
SourceHost: f.DomainHost,
|
||||
})
|
||||
}
|
||||
|
||||
if result == nil {
|
||||
result = []FeedDeniedInfo{}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(result)
|
||||
}
|
||||
|
||||
func (s *PublisherService) handlePublishCandidates(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 50
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
fmt.Sscanf(l, "%d", &limit)
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
}
|
||||
|
||||
feeds, err := s.GetPublishCandidates(limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
type CandidateInfo struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Category string `json:"category"`
|
||||
SourceHost string `json:"source_host"`
|
||||
ItemCount int `json:"item_count"`
|
||||
DerivedHandle string `json:"derived_handle"`
|
||||
}
|
||||
|
||||
var result []CandidateInfo
|
||||
for _, f := range feeds {
|
||||
result = append(result, CandidateInfo{
|
||||
URL: f.URL,
|
||||
Title: f.Title,
|
||||
Category: f.Category,
|
||||
SourceHost: f.DomainHost,
|
||||
ItemCount: f.ItemCount,
|
||||
DerivedHandle: shared.DeriveHandleFromFeed(f.URL),
|
||||
})
|
||||
}
|
||||
|
||||
if result == nil {
|
||||
result = []CandidateInfo{}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(result)
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleUnpublishedItems(w http.ResponseWriter, r *http.Request) {
|
||||
feedURL := r.URL.Query().Get("url")
|
||||
if feedURL == "" {
|
||||
http.Error(w, "url parameter required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
limit := 50
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
fmt.Sscanf(l, "%d", &limit)
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
}
|
||||
|
||||
items, err := s.GetUnpublishedItems(feedURL, limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if items == nil {
|
||||
items = []*shared.Item{}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(items)
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleTestPublish(w http.ResponseWriter, r *http.Request) {
|
||||
feedURL := r.URL.Query().Get("feedUrl")
|
||||
guidParam := r.URL.Query().Get("guid")
|
||||
handle := r.URL.Query().Get("handle")
|
||||
password := r.URL.Query().Get("password")
|
||||
pdsHost := r.URL.Query().Get("pds")
|
||||
|
||||
if feedURL == "" || guidParam == "" {
|
||||
http.Error(w, "feedUrl and guid parameters required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if handle == "" || password == "" {
|
||||
http.Error(w, "handle and password parameters required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if pdsHost == "" {
|
||||
pdsHost = s.pdsHost
|
||||
}
|
||||
|
||||
item, err := s.GetItemByGUID(feedURL, guidParam)
|
||||
if err != nil {
|
||||
http.Error(w, "item not found: "+err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
publisher := NewPublisher(pdsHost)
|
||||
session, err := publisher.CreateSession(handle, password)
|
||||
if err != nil {
|
||||
http.Error(w, "auth failed: "+err.Error(), http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
uri, err := publisher.PublishItem(session, item)
|
||||
if err != nil {
|
||||
http.Error(w, "publish failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
s.MarkItemPublished(item.FeedURL, item.GUID, uri)
|
||||
|
||||
rkeyTime := item.PubDate
|
||||
if rkeyTime.IsZero() {
|
||||
rkeyTime = item.DiscoveredAt
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "published",
|
||||
"uri": uri,
|
||||
"feedUrl": item.FeedURL,
|
||||
"guid": item.GUID,
|
||||
"title": item.Title,
|
||||
"rkey": GenerateRkey(item.GUID, rkeyTime),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handlePublishFeed(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "not_implemented",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handlePublishFeedFull(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "not_implemented",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleCreateAccount(w http.ResponseWriter, r *http.Request) {
|
||||
handle := r.URL.Query().Get("handle")
|
||||
email := r.URL.Query().Get("email")
|
||||
password := r.URL.Query().Get("password")
|
||||
pdsHost := r.URL.Query().Get("pds")
|
||||
inviteCode := r.URL.Query().Get("inviteCode")
|
||||
pdsAdminPassword := r.URL.Query().Get("pdsAdminPassword")
|
||||
|
||||
if handle == "" || password == "" {
|
||||
http.Error(w, "handle and password parameters required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if pdsHost == "" {
|
||||
pdsHost = s.pdsHost
|
||||
}
|
||||
if email == "" {
|
||||
email = handle + "@1440.news"
|
||||
}
|
||||
|
||||
publisher := NewPublisher(pdsHost)
|
||||
|
||||
if pdsAdminPassword != "" && inviteCode == "" {
|
||||
code, err := publisher.CreateInviteCode(pdsAdminPassword, 1)
|
||||
if err != nil {
|
||||
http.Error(w, "create invite failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
inviteCode = code
|
||||
}
|
||||
|
||||
session, err := publisher.CreateAccount(handle, email, password, inviteCode)
|
||||
if err != nil {
|
||||
http.Error(w, "create account failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if err := publisher.FollowAsDirectory(session.DID); err != nil {
|
||||
fmt.Printf("API: directory follow failed for %s: %v\n", handle, err)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "created",
|
||||
"handle": session.Handle,
|
||||
"did": session.DID,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleUpdateProfile(w http.ResponseWriter, r *http.Request) {
|
||||
handle := r.URL.Query().Get("handle")
|
||||
password := r.URL.Query().Get("password")
|
||||
pdsHost := r.URL.Query().Get("pds")
|
||||
displayName := r.URL.Query().Get("displayName")
|
||||
description := r.URL.Query().Get("description")
|
||||
faviconURL := r.URL.Query().Get("faviconUrl")
|
||||
|
||||
if handle == "" || password == "" {
|
||||
http.Error(w, "handle and password parameters required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if pdsHost == "" {
|
||||
pdsHost = s.pdsHost
|
||||
}
|
||||
|
||||
publisher := NewPublisher(pdsHost)
|
||||
|
||||
session, err := publisher.CreateSession(handle, password)
|
||||
if err != nil {
|
||||
http.Error(w, "auth failed: "+err.Error(), http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
var avatar *BlobRef
|
||||
if faviconURL != "" {
|
||||
faviconData, mimeType, err := FetchFaviconBytes(faviconURL)
|
||||
if err != nil {
|
||||
http.Error(w, "fetch favicon failed: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
avatar, err = publisher.UploadBlob(session, faviconData, mimeType)
|
||||
if err != nil {
|
||||
http.Error(w, "upload favicon failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := publisher.UpdateProfile(session, displayName, description, avatar); err != nil {
|
||||
http.Error(w, "update profile failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "updated",
|
||||
"handle": handle,
|
||||
"displayName": displayName,
|
||||
"hasAvatar": avatar != nil,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleDeriveHandle(w http.ResponseWriter, r *http.Request) {
|
||||
feedURL := r.URL.Query().Get("url")
|
||||
if feedURL == "" {
|
||||
http.Error(w, "url parameter required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
handle := shared.DeriveHandleFromFeed(feedURL)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"url": feedURL,
|
||||
"handle": handle,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleResetAllPublishing(w http.ResponseWriter, r *http.Request) {
|
||||
accountsCleared, err := s.db.Exec(`UPDATE feeds SET publish_account = NULL WHERE publish_account IS NOT NULL`)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
itemsCleared, err := s.db.Exec(`UPDATE items SET published_at = NULL WHERE published_at IS NOT NULL`)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
statusReset, err := s.db.Exec(`UPDATE feeds SET publish_status = 'hold' WHERE publish_status IS NOT NULL`)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"success": true,
|
||||
"accounts_cleared": accountsCleared,
|
||||
"items_cleared": itemsCleared,
|
||||
"status_reset": statusReset,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PublisherService) handleRefreshProfiles(w http.ResponseWriter, r *http.Request) {
|
||||
password := r.URL.Query().Get("password")
|
||||
pdsHost := r.URL.Query().Get("pds")
|
||||
|
||||
if password == "" {
|
||||
http.Error(w, "password parameter required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if pdsHost == "" {
|
||||
pdsHost = s.pdsHost
|
||||
}
|
||||
|
||||
publisher := NewPublisher(pdsHost)
|
||||
s.RefreshAllProfiles(publisher, password)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "profiles refreshed",
|
||||
})
|
||||
}
|
||||
|
||||
// Database helper methods
|
||||
|
||||
func (s *PublisherService) SetPublishStatus(feedURL, status, account string) error {
|
||||
feedURL = shared.NormalizeURL(feedURL)
|
||||
|
||||
if status == "pass" && account == "" {
|
||||
account = shared.DeriveHandleFromFeed(feedURL)
|
||||
}
|
||||
|
||||
_, err := s.db.Exec(`
|
||||
UPDATE feeds SET publish_status = $1, publish_account = $2 WHERE url = $3
|
||||
`, status, shared.NullableString(account), feedURL)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *PublisherService) GetFeedsByPublishStatus(status string) ([]*shared.Feed, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT url, type, category, title, description, language, site_url,
|
||||
discovered_at, last_checked_at, next_check_at, last_build_date,
|
||||
etag, last_modified,
|
||||
status, last_error, last_error_at,
|
||||
source_url, domain_host, domain_tld,
|
||||
item_count, oldest_item_date, newest_item_date,
|
||||
no_update,
|
||||
publish_status, publish_account
|
||||
FROM feeds
|
||||
WHERE publish_status = $1
|
||||
`, status)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanFeeds(rows)
|
||||
}
|
||||
|
||||
func (s *PublisherService) GetPublishCandidates(limit int) ([]*shared.Feed, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT url, type, category, title, description, language, site_url,
|
||||
discovered_at, last_checked_at, next_check_at, last_build_date,
|
||||
etag, last_modified,
|
||||
status, last_error, last_error_at,
|
||||
source_url, domain_host, domain_tld,
|
||||
item_count, oldest_item_date, newest_item_date,
|
||||
no_update,
|
||||
publish_status, publish_account
|
||||
FROM feeds
|
||||
WHERE publish_status = 'hold' AND item_count > 0 AND status = 'pass'
|
||||
ORDER BY item_count DESC
|
||||
LIMIT $1
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanFeeds(rows)
|
||||
}
|
||||
|
||||
func (s *PublisherService) GetUnpublishedItemCount(feedURL string) (int, error) {
|
||||
var count int
|
||||
err := s.db.QueryRow(`
|
||||
SELECT COUNT(*) FROM items WHERE feed_url = $1 AND published_at IS NULL
|
||||
`, feedURL).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (s *PublisherService) GetItemByGUID(feedURL, guid string) (*shared.Item, error) {
|
||||
items, err := s.db.Query(`
|
||||
SELECT feed_url, guid, title, link, description, content, author, pub_date, discovered_at, updated_at,
|
||||
enclosure_url, enclosure_type, enclosure_length, image_urls, tags,
|
||||
published_at, published_uri
|
||||
FROM items
|
||||
WHERE feed_url = $1 AND guid = $2
|
||||
`, feedURL, guid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer items.Close()
|
||||
|
||||
result, err := scanItems(items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return nil, fmt.Errorf("item not found")
|
||||
}
|
||||
return result[0], nil
|
||||
}
|
||||
|
||||
func (s *PublisherService) RefreshAllProfiles(publisher *Publisher, feedPassword string) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT url, title, description, site_url, domain_host as source_host, publish_account
|
||||
FROM feeds
|
||||
WHERE publish_account IS NOT NULL AND publish_account <> ''
|
||||
`)
|
||||
if err != nil {
|
||||
fmt.Printf("RefreshProfiles: query error: %v\n", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var feedURL, account string
|
||||
var title, description, siteURL, sourceHost *string
|
||||
if err := rows.Scan(&feedURL, &title, &description, &siteURL, &sourceHost, &account); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
session, err := publisher.CreateSession(account, feedPassword)
|
||||
if err != nil {
|
||||
fmt.Printf("RefreshProfiles: login failed for %s: %v\n", account, err)
|
||||
continue
|
||||
}
|
||||
|
||||
displayName := shared.StringValue(title)
|
||||
if displayName == "" {
|
||||
displayName = account
|
||||
}
|
||||
desc := stripHTML(shared.StringValue(description))
|
||||
if desc == "" {
|
||||
desc = "News feed via 1440.news"
|
||||
}
|
||||
feedURLFull := "https://" + feedURL
|
||||
desc = feedURLFull + "\n\n" + desc
|
||||
|
||||
if len(displayName) > 64 {
|
||||
displayName = displayName[:61] + "..."
|
||||
}
|
||||
if len(desc) > 256 {
|
||||
desc = desc[:253] + "..."
|
||||
}
|
||||
|
||||
var avatar *BlobRef
|
||||
faviconSource := shared.StringValue(siteURL)
|
||||
if faviconSource == "" {
|
||||
faviconSource = shared.StringValue(sourceHost)
|
||||
}
|
||||
if faviconSource != "" {
|
||||
faviconData, mimeType, err := FetchFaviconBytes(faviconSource)
|
||||
if err == nil && len(faviconData) > 0 {
|
||||
avatar, _ = publisher.UploadBlob(session, faviconData, mimeType)
|
||||
}
|
||||
}
|
||||
|
||||
if err := publisher.UpdateProfile(session, displayName, desc, avatar); err != nil {
|
||||
fmt.Printf("RefreshProfiles: update failed for %s: %v\n", account, err)
|
||||
} else {
|
||||
fmt.Printf("RefreshProfiles: updated %s\n", account)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scanFeeds helper
|
||||
func scanFeeds(rows interface{ Next() bool; Scan(...interface{}) error; Err() error }) ([]*shared.Feed, error) {
|
||||
var feeds []*shared.Feed
|
||||
|
||||
for rows.Next() {
|
||||
feed := &shared.Feed{}
|
||||
var feedType, category, title, description, language, siteURL *string
|
||||
var lastCheckedAt, nextCheckAt, lastBuildDate *interface{}
|
||||
var etag, lastModified, lastError, sourceURL, domainTLD *string
|
||||
var lastErrorAt *interface{}
|
||||
var oldestItemDate, newestItemDate *interface{}
|
||||
var publishStatus, publishAccount *string
|
||||
|
||||
err := rows.Scan(
|
||||
&feed.URL, &feedType, &category, &title, &description, &language, &siteURL,
|
||||
&feed.DiscoveredAt, &lastCheckedAt, &nextCheckAt, &lastBuildDate,
|
||||
&etag, &lastModified,
|
||||
&feed.Status, &lastError, &lastErrorAt,
|
||||
&sourceURL, &feed.DomainHost, &domainTLD,
|
||||
&feed.ItemCount, &oldestItemDate, &newestItemDate,
|
||||
&feed.NoUpdate,
|
||||
&publishStatus, &publishAccount,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
feed.Type = shared.StringValue(feedType)
|
||||
feed.Category = shared.StringValue(category)
|
||||
feed.Title = shared.StringValue(title)
|
||||
feed.Description = shared.StringValue(description)
|
||||
feed.Language = shared.StringValue(language)
|
||||
feed.SiteURL = shared.StringValue(siteURL)
|
||||
feed.ETag = shared.StringValue(etag)
|
||||
feed.LastModified = shared.StringValue(lastModified)
|
||||
feed.LastError = shared.StringValue(lastError)
|
||||
feed.SourceURL = shared.StringValue(sourceURL)
|
||||
feed.DomainTLD = shared.StringValue(domainTLD)
|
||||
feed.PublishStatus = shared.StringValue(publishStatus)
|
||||
feed.PublishAccount = shared.StringValue(publishAccount)
|
||||
|
||||
feeds = append(feeds, feed)
|
||||
}
|
||||
|
||||
return feeds, rows.Err()
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Try to load pds.env if it exists
|
||||
if data, err := os.ReadFile("pds.env"); err == nil {
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, "=", 2)
|
||||
if len(parts) == 2 {
|
||||
key := strings.TrimSpace(parts[0])
|
||||
value := strings.TrimSpace(parts[1])
|
||||
if os.Getenv(key) == "" {
|
||||
os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user