commit f1e68d42fc8761a35a2e8bf5edf895ce6345c59d Author: primal Date: Mon Feb 2 15:30:15 2026 -0500 Initial commit diff --git a/.launch.sh b/.launch.sh new file mode 100755 index 0000000..6e84d32 --- /dev/null +++ b/.launch.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "$0")" + +~/apps/.launch.sh "$@" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..904a0d1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM golang:latest AS builder + +WORKDIR /build + +# Copy shared module first +COPY commons/ ./commons/ + +# Copy publisher module +COPY publisher/ ./publisher/ + +# Build the binary +WORKDIR /build/publisher +RUN go mod download +RUN CGO_ENABLED=0 go build -o publisher . + +# Runtime stage +FROM ubuntu:latest + +WORKDIR /app + +# Install runtime dependencies +RUN apt-get update && apt-get install -y ca-certificates tzdata curl wget && rm -rf /var/lib/apt/lists/* + +# Copy binary from builder +COPY --from=builder /build/publisher/publisher . + +EXPOSE 4322 + +CMD ["./publisher"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0765439 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +services: + publisher: + build: + context: .. + dockerfile: publisher/Dockerfile + image: atproto-1440news-publisher + container_name: atproto-1440news-publisher + restart: unless-stopped + stop_grace_period: 30s + env_file: + - pds.env + environment: + DB_HOST: infra-postgres + DB_PORT: 5432 + DB_USER: dba_1440_news + DB_PASSWORD_FILE: /run/secrets/db_password + DB_NAME: db_1440_news + secrets: + - db_password + networks: + - proxy + - atproto + labels: + - "traefik.enable=true" + # Production: HTTPS with Let's Encrypt for api.1440.news + - "traefik.http.routers.publisher-1440.rule=Host(`api.1440.news`)" + - "traefik.http.routers.publisher-1440.entrypoints=https" + - "traefik.http.routers.publisher-1440.tls.certresolver=letsencrypt-dns" + # Production: HTTP to HTTPS redirect + - "traefik.http.routers.publisher-1440-redirect.rule=Host(`api.1440.news`)" + - "traefik.http.routers.publisher-1440-redirect.entrypoints=http" + - "traefik.http.routers.publisher-1440-redirect.middlewares=https-redirect" + - "traefik.http.middlewares.https-redirect.redirectscheme.scheme=https" + - "traefik.http.middlewares.https-redirect.redirectscheme.permanent=true" + # Local development + - "traefik.http.routers.publisher-1440-local.rule=Host(`api.1440.localhost`)" + - "traefik.http.routers.publisher-1440-local.entrypoints=http" + # Shared service + - "traefik.http.services.publisher-1440.loadbalancer.server.port=4322" + +secrets: + db_password: + file: ../../../infra/postgres/secrets/dba_1440_news_password.txt + +networks: + proxy: + external: true + atproto: + external: true diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..dc04cc5 --- /dev/null +++ b/go.mod @@ -0,0 +1,26 @@ +module github.com/1440news/publisher + +go 1.24.0 + +require ( + github.com/1440news/commons v0.0.0 + github.com/jackc/pgx/v5 v5.7.5 + go.deanishe.net/favicon v0.1.0 + golang.org/x/image v0.26.0 +) + +require ( + github.com/PuerkitoBio/goquery v1.6.0 // indirect + github.com/andybalholm/cascadia v1.1.0 // indirect + github.com/friendsofgo/errors v0.9.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect +) + +replace github.com/1440news/commons => ../commons diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4d3bdec --- /dev/null +++ b/go.sum @@ -0,0 +1,144 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= +github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk= +github.com/friendsofgo/errors v0.9.2/go.mod h1:yCvFW5AkDIL9qn7suHVLiI/gH228n7PC4Pn44IGoTOI= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/strfmt v0.19.8/go.mod h1:qBBipho+3EoIqn6YDI+4RnQEtj6jT/IdKm+PAlXxSUc= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +go.deanishe.net/favicon v0.1.0 h1:Afy941gjRik+DjUUcYHUxcztFEeFse2ITBkMMOlgefM= +go.deanishe.net/favicon v0.1.0/go.mod h1:vIKVI+lUh8k3UAzaN4gjC+cpyatLQWmx0hVX4vLE8jU= +go.mongodb.org/mongo-driver v1.4.2/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/image v0.26.0 h1:4XjIFEZWQmCZi6Wv8BoxsDhRU3RVnLX04dToTDAEPlY= +golang.org/x/image v0.26.0/go.mod h1:lcxbMFAovzpnJxzXS3nyL83K27tmqtKzIJpctK8YO5c= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/image.go b/image.go new file mode 100644 index 0000000..094f24b --- /dev/null +++ b/image.go @@ -0,0 +1,306 @@ +package main + +import ( + "bytes" + "fmt" + "image" + _ "image/gif" + "image/jpeg" + _ "image/png" + "io" + "net/http" + "net/url" + "strings" + "time" + + "go.deanishe.net/favicon" + "golang.org/x/image/draw" + _ "golang.org/x/image/webp" +) + +// ImageUploadResult contains the uploaded blob and image dimensions +type ImageUploadResult struct { + Blob *BlobRef + Width int + Height int +} + +// FetchFavicon tries to get a favicon URL for a site +func (p *Publisher) FetchFavicon(siteURL string) string { + if siteURL == "" { + return "" + } + + if !strings.Contains(siteURL, "://") { + siteURL = "https://" + siteURL + } + u, err := url.Parse(siteURL) + if err != nil { + return "" + } + + finder := favicon.New( + favicon.WithClient(p.httpClient), + ) + + icons, err := finder.Find(siteURL) + if err == nil && len(icons) > 0 { + var bestIcon string + var bestScore int + + for _, icon := range icons { + if icon.Width > 0 && icon.Width < 32 { + continue + } + + lowerURL := strings.ToLower(icon.URL) + if strings.Contains(lowerURL, "og-image") || strings.Contains(lowerURL, "og_image") || + strings.Contains(lowerURL, "opengraph") || strings.Contains(lowerURL, "twitter") { + continue + } + + if icon.Width > 0 && icon.Height > 0 { + ratio := float64(icon.Width) / float64(icon.Height) + if ratio > 1.5 || ratio < 0.67 { + continue + } + } + + score := 0 + + if strings.Contains(lowerURL, "favicon") || strings.Contains(lowerURL, "icon") || + strings.Contains(lowerURL, "apple-touch") { + score += 100 + } + + if icon.MimeType == "image/png" { + score += 50 + } else if icon.MimeType == "image/x-icon" || strings.HasSuffix(lowerURL, ".ico") { + score += 40 + } else if icon.MimeType == "image/jpeg" { + score += 10 + } + + if icon.Width >= 64 && icon.Width <= 512 { + score += 30 + } else if icon.Width > 0 { + score += 10 + } + + if score > bestScore { + bestScore = score + bestIcon = icon.URL + } + } + + if bestIcon != "" { + return bestIcon + } + + for _, icon := range icons { + lowerURL := strings.ToLower(icon.URL) + if !strings.Contains(lowerURL, "og-image") && !strings.Contains(lowerURL, "og_image") { + return icon.URL + } + } + } + + return fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host) +} + +func (p *Publisher) fetchAndUploadImage(session *PDSSession, imageURL string) *BlobRef { + result := p.fetchAndUploadImageWithDimensions(session, imageURL) + if result == nil { + return nil + } + return result.Blob +} + +func upgradeImageURL(imageURL string) string { + if strings.Contains(imageURL, "ichef.bbci.co.uk") { + imageURL = strings.Replace(imageURL, "/standard/240/", "/standard/800/", 1) + imageURL = strings.Replace(imageURL, "/standard/480/", "/standard/800/", 1) + } + return imageURL +} + +func (p *Publisher) fetchAndUploadImageWithDimensions(session *PDSSession, imageURL string) *ImageUploadResult { + imageURL = upgradeImageURL(imageURL) + + resp, err := p.httpClient.Get(imageURL) + if err != nil { + return nil + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil + } + + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + if strings.HasSuffix(strings.ToLower(imageURL), ".png") { + contentType = "image/png" + } else if strings.HasSuffix(strings.ToLower(imageURL), ".gif") { + contentType = "image/gif" + } else if strings.HasSuffix(strings.ToLower(imageURL), ".webp") { + contentType = "image/webp" + } else { + contentType = "image/jpeg" + } + } + + if !strings.HasPrefix(contentType, "image/") { + return nil + } + + data, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) + if err != nil || len(data) == 0 { + return nil + } + + imgConfig, _, err := image.DecodeConfig(bytes.NewReader(data)) + width, height := 1, 1 + if err == nil { + width, height = imgConfig.Width, imgConfig.Height + } + + const maxBlobSize = 900 * 1024 + + if len(data) > maxBlobSize { + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil + } + + scaleFactor := 0.9 + + for attempt := 0; attempt < 5; attempt++ { + newWidth := int(float64(width) * scaleFactor) + newHeight := int(float64(height) * scaleFactor) + + if newWidth < 100 { + newWidth = 100 + } + if newHeight < 100 { + newHeight = 100 + } + + resized := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) + draw.CatmullRom.Scale(resized, resized.Bounds(), img, img.Bounds(), draw.Over, nil) + + var buf bytes.Buffer + if err := jpeg.Encode(&buf, resized, &jpeg.Options{Quality: 85}); err != nil { + return nil + } + + if buf.Len() <= maxBlobSize { + data = buf.Bytes() + width = newWidth + height = newHeight + contentType = "image/jpeg" + break + } + + scaleFactor *= 0.8 + } + + if len(data) > maxBlobSize { + return nil + } + } + + blob, err := p.UploadBlob(session, data, contentType) + if err != nil { + return nil + } + + return &ImageUploadResult{ + Blob: blob, + Width: width, + Height: height, + } +} + +// FetchFaviconBytes downloads a favicon/icon from a URL +func FetchFaviconBytes(siteURL string) ([]byte, string, error) { + if !strings.HasPrefix(siteURL, "http") { + siteURL = "https://" + siteURL + } + + u, err := url.Parse(siteURL) + if err != nil { + return nil, "", err + } + + client := &http.Client{Timeout: 10 * time.Second} + + finder := favicon.New( + favicon.WithClient(client), + favicon.IgnoreNoSize, + ) + + icons, err := finder.Find(siteURL) + if err != nil || len(icons) == 0 { + googleURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host) + return fetchIconBytes(client, googleURL) + } + + var iconURLs []string + for _, icon := range icons { + if icon.Width > 0 && icon.Width < 32 { + continue + } + if icon.MimeType == "image/png" || icon.MimeType == "image/jpeg" { + iconURLs = append([]string{icon.URL}, iconURLs...) + } else { + iconURLs = append(iconURLs, icon.URL) + } + } + + if len(iconURLs) == 0 { + for _, icon := range icons { + iconURLs = append(iconURLs, icon.URL) + } + } + + for _, iconURL := range iconURLs { + data, mimeType, err := fetchIconBytes(client, iconURL) + if err == nil && len(data) > 0 { + return data, mimeType, nil + } + } + + googleURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host) + return fetchIconBytes(client, googleURL) +} + +func fetchIconBytes(client *http.Client, iconURL string) ([]byte, string, error) { + resp, err := client.Get(iconURL) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("HTTP %d", resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, "", err + } + + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + if strings.HasSuffix(iconURL, ".png") { + contentType = "image/png" + } else if strings.HasSuffix(iconURL, ".ico") { + contentType = "image/x-icon" + } else { + contentType = "image/png" + } + } + + return data, contentType, nil +} diff --git a/items.go b/items.go new file mode 100644 index 0000000..4a8bf04 --- /dev/null +++ b/items.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + + "github.com/1440news/commons" + "github.com/jackc/pgx/v5" +) + +// scanItems scans rows into Item structs +func scanItems(rows pgx.Rows) ([]*shared.Item, error) { + var items []*shared.Item + + for rows.Next() { + item := &shared.Item{} + var guid, title, link, description, content, author *string + var pubDate, updatedAt, publishedAt *interface{} + var publishedUri *string + var enclosureUrl, enclosureType *string + var enclosureLength *int64 + var imageUrlsJSON, tagsJSON *string + + err := rows.Scan( + &item.FeedURL, &guid, &title, &link, + &description, &content, &author, &pubDate, + &item.DiscoveredAt, &updatedAt, + &enclosureUrl, &enclosureType, &enclosureLength, + &imageUrlsJSON, &tagsJSON, + &publishedAt, &publishedUri, + ) + if err != nil { + continue + } + + item.GUID = shared.StringValue(guid) + item.Title = shared.StringValue(title) + item.Link = shared.StringValue(link) + item.Description = shared.StringValue(description) + item.Content = shared.StringValue(content) + item.Author = shared.StringValue(author) + item.PublishedUri = shared.StringValue(publishedUri) + + if pubDate != nil { + if t, ok := (*pubDate).(interface{ Time() interface{} }); ok { + if tm, ok := t.Time().(interface{ IsZero() bool }); ok && !tm.IsZero() { + // Handle time conversion + } + } + } + + if enclosureUrl != nil && *enclosureUrl != "" { + item.Enclosure = &shared.Enclosure{ + URL: *enclosureUrl, + Type: shared.StringValue(enclosureType), + } + if enclosureLength != nil { + item.Enclosure.Length = *enclosureLength + } + } + + if imageUrlsJSON != nil && *imageUrlsJSON != "" { + json.Unmarshal([]byte(*imageUrlsJSON), &item.ImageURLs) + } + + if tagsJSON != nil && *tagsJSON != "" { + json.Unmarshal([]byte(*tagsJSON), &item.Tags) + } + + items = append(items, item) + } + + return items, rows.Err() +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..5bb9582 --- /dev/null +++ b/main.go @@ -0,0 +1,211 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/1440news/commons" +) + +func main() { + fmt.Println("Starting publisher service...") + + // Open database connection + db, err := shared.OpenDatabase("") + if err != nil { + log.Fatalf("Failed to connect to database: %v", err) + } + defer db.Close() + + // Load PDS configuration + pdsHost := os.Getenv("PDS_HOST") + if pdsHost == "" { + pdsHost = "https://pds.1440.news" + } + + feedPassword := os.Getenv("FEED_PASSWORD") + if feedPassword == "" { + feedPassword = "feed1440!" + } + + // Create publisher service + pub := NewPublisherService(db, pdsHost, feedPassword) + + // Start HTTP server + go func() { + if err := pub.StartServer("0.0.0.0:4322"); err != nil { + log.Fatalf("Server error: %v", err) + } + }() + + // Start publish loop + go pub.StartPublishLoop() + + fmt.Printf("Publisher service running (PDS: %s)\n", pdsHost) + + // Wait for shutdown signal + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + <-sigChan + + fmt.Println("Shutting down publisher service...") +} + +// loadEnvFile loads environment variables from a file (e.g., pds.env) +func loadEnvFile(filename string) { + data, err := os.ReadFile(filename) + if err != nil { + return // File doesn't exist or can't be read + } + + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + if os.Getenv(key) == "" { + os.Setenv(key, value) + } + } + } +} + +// PublisherService manages publishing items to AT Protocol PDS +type PublisherService struct { + db *shared.DB + publisher *Publisher + pdsHost string + feedPassword string +} + +// NewPublisherService creates a new publisher service +func NewPublisherService(db *shared.DB, pdsHost, feedPassword string) *PublisherService { + return &PublisherService{ + db: db, + publisher: NewPublisher(pdsHost), + pdsHost: pdsHost, + feedPassword: feedPassword, + } +} + +// StartPublishLoop runs the automatic publishing loop +func (s *PublisherService) StartPublishLoop() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + // Run immediately on start + s.publishPendingItems() + + for range ticker.C { + s.publishPendingItems() + } +} + +// publishPendingItems publishes unpublished items for all enabled feeds +func (s *PublisherService) publishPendingItems() { + // Get all feeds with publish_status = 'pass' + rows, err := s.db.Query(` + SELECT url, publish_account + FROM feeds + WHERE publish_status = 'pass' AND publish_account IS NOT NULL + `) + if err != nil { + fmt.Printf("Publish loop: query error: %v\n", err) + return + } + defer rows.Close() + + var feeds []struct { + URL string + Account string + } + + for rows.Next() { + var url, account string + if err := rows.Scan(&url, &account); err != nil { + continue + } + feeds = append(feeds, struct { + URL string + Account string + }{url, account}) + } + + if len(feeds) == 0 { + return + } + + fmt.Printf("Publish loop: checking %d feeds\n", len(feeds)) + + for _, feed := range feeds { + s.publishFeedItems(feed.URL, feed.Account) + } +} + +// publishFeedItems publishes unpublished items for a single feed +func (s *PublisherService) publishFeedItems(feedURL, account string) { + // Get unpublished items (limit to 5 per cycle to avoid overwhelming) + items, err := s.GetUnpublishedItems(feedURL, 5) + if err != nil || len(items) == 0 { + return + } + + // Authenticate with the feed account + session, err := s.publisher.CreateSession(account, s.feedPassword) + if err != nil { + fmt.Printf("Publish: auth failed for %s: %v\n", account, err) + return + } + + for _, item := range items { + uri, err := s.publisher.PublishItem(session, item) + if err != nil { + fmt.Printf("Publish: failed to publish %s: %v\n", item.GUID, err) + continue + } + + if err := s.MarkItemPublished(item.FeedURL, item.GUID, uri); err != nil { + fmt.Printf("Publish: failed to mark published %s: %v\n", item.GUID, err) + } + + // Small delay between posts + time.Sleep(1100 * time.Millisecond) + } +} + +// GetUnpublishedItems returns unpublished items for a feed +func (s *PublisherService) GetUnpublishedItems(feedURL string, limit int) ([]*shared.Item, error) { + rows, err := s.db.Query(` + SELECT feed_url, guid, title, link, description, content, author, pub_date, discovered_at, updated_at, + enclosure_url, enclosure_type, enclosure_length, image_urls, tags, + published_at, published_uri + FROM items + WHERE feed_url = $1 AND published_at IS NULL + ORDER BY pub_date ASC + LIMIT $2 + `, feedURL, limit) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanItems(rows) +} + +// MarkItemPublished marks an item as published with the given URI +func (s *PublisherService) MarkItemPublished(feedURL, guid, uri string) error { + _, err := s.db.Exec(` + UPDATE items SET published_at = NOW(), published_uri = $1 WHERE feed_url = $2 AND guid = $3 + `, uri, feedURL, guid) + return err +} diff --git a/pds_auth.go b/pds_auth.go new file mode 100644 index 0000000..a3a4e8c --- /dev/null +++ b/pds_auth.go @@ -0,0 +1,183 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "time" +) + +// CreateSession authenticates with the PDS and returns a session +func (p *Publisher) CreateSession(handle, password string) (*PDSSession, error) { + payload := map[string]string{ + "identifier": handle, + "password": password, + } + body, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + resp, err := p.httpClient.Post( + p.pdsHost+"/xrpc/com.atproto.server.createSession", + "application/json", + bytes.NewReader(body), + ) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("auth failed: %s - %s", resp.Status, string(respBody)) + } + + var session PDSSession + if err := json.NewDecoder(resp.Body).Decode(&session); err != nil { + return nil, err + } + + return &session, nil +} + +// CreateAccount creates a new account on the PDS +func (p *Publisher) CreateAccount(handle, email, password, inviteCode string) (*PDSSession, error) { + payload := map[string]interface{}{ + "handle": handle, + "email": email, + "password": password, + } + if inviteCode != "" { + payload["inviteCode"] = inviteCode + } + + body, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + resp, err := p.httpClient.Post( + p.pdsHost+"/xrpc/com.atproto.server.createAccount", + "application/json", + bytes.NewReader(body), + ) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("create account failed: %s - %s", resp.Status, string(respBody)) + } + + var session PDSSession + if err := json.Unmarshal(respBody, &session); err != nil { + return nil, err + } + + return &session, nil +} + +// CreateInviteCode creates an invite code using PDS admin password (Basic Auth) +func (p *Publisher) CreateInviteCode(adminPassword string, useCount int) (string, error) { + payload := map[string]interface{}{ + "useCount": useCount, + } + + body, err := json.Marshal(payload) + if err != nil { + return "", err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.server.createInviteCode", bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth("admin", adminPassword) + + resp, err := p.httpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("create invite failed: %s - %s", resp.Status, string(respBody)) + } + + var result struct { + Code string `json:"code"` + } + if err := json.Unmarshal(respBody, &result); err != nil { + return "", err + } + + return result.Code, nil +} + +// FollowAccount creates a follow record from the authenticated session to the target DID +func (p *Publisher) FollowAccount(session *PDSSession, targetDID string) error { + now := time.Now().UTC().Format(time.RFC3339) + record := map[string]interface{}{ + "$type": "app.bsky.graph.follow", + "subject": targetDID, + "createdAt": now, + } + + payload := map[string]interface{}{ + "repo": session.DID, + "collection": "app.bsky.graph.follow", + "record": record, + } + + body, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.createRecord", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + resp, err := p.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("follow failed: %s - %s", resp.Status, string(respBody)) + } + + return nil +} + +// FollowAsDirectory logs in as the directory account and follows the target DID +func (p *Publisher) FollowAsDirectory(targetDID string) error { + dirHandle := os.Getenv("DIRECTORY_HANDLE") + dirPassword := os.Getenv("DIRECTORY_PASSWORD") + + if dirHandle == "" || dirPassword == "" { + return nil + } + + session, err := p.CreateSession(dirHandle, dirPassword) + if err != nil { + return fmt.Errorf("directory login failed: %w", err) + } + + return p.FollowAccount(session, targetDID) +} diff --git a/pds_records.go b/pds_records.go new file mode 100644 index 0000000..4337afb --- /dev/null +++ b/pds_records.go @@ -0,0 +1,342 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +// BlobRef represents a blob reference for profile images +type BlobRef struct { + Type string `json:"$type"` + Ref Link `json:"ref"` + MimeType string `json:"mimeType"` + Size int64 `json:"size"` +} + +type Link struct { + Link string `json:"$link"` +} + +// UploadBlob uploads an image to the PDS and returns a blob reference +func (p *Publisher) UploadBlob(session *PDSSession, data []byte, mimeType string) (*BlobRef, error) { + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.uploadBlob", bytes.NewReader(data)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", mimeType) + req.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + resp, err := p.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("upload blob failed: %s - %s", resp.Status, string(respBody)) + } + + var result struct { + Blob BlobRef `json:"blob"` + } + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, err + } + + return &result.Blob, nil +} + +// UpdateProfile updates the profile for an account +func (p *Publisher) UpdateProfile(session *PDSSession, displayName, description string, avatar *BlobRef) error { + getReq, err := http.NewRequest("GET", + p.pdsHost+"/xrpc/com.atproto.repo.getRecord?repo="+session.DID+"&collection=app.bsky.actor.profile&rkey=self", + nil) + if err != nil { + return err + } + getReq.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + getResp, err := p.httpClient.Do(getReq) + + var existingCID string + profile := map[string]interface{}{ + "$type": "app.bsky.actor.profile", + } + + if err == nil && getResp.StatusCode == http.StatusOK { + defer getResp.Body.Close() + var existing struct { + CID string `json:"cid"` + Value map[string]interface{} `json:"value"` + } + if json.NewDecoder(getResp.Body).Decode(&existing) == nil { + existingCID = existing.CID + profile = existing.Value + } + } else if getResp != nil { + getResp.Body.Close() + } + + if displayName != "" { + profile["displayName"] = displayName + } + if description != "" { + profile["description"] = description + } + if avatar != nil { + profile["avatar"] = avatar + } + + payload := map[string]interface{}{ + "repo": session.DID, + "collection": "app.bsky.actor.profile", + "rkey": "self", + "record": profile, + } + if existingCID != "" { + payload["swapRecord"] = existingCID + } + + body, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.putRecord", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + resp, err := p.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("update profile failed: %s - %s", resp.Status, string(respBody)) + } + + return nil +} + +// DeleteAllPosts deletes all posts from an account +func (p *Publisher) DeleteAllPosts(session *PDSSession) (int, error) { + deleted := 0 + cursor := "" + + for { + listURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=app.bsky.feed.post&limit=100", + p.pdsHost, session.DID) + if cursor != "" { + listURL += "&cursor=" + url.QueryEscape(cursor) + } + + req, err := http.NewRequest("GET", listURL, nil) + if err != nil { + return deleted, err + } + req.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + resp, err := p.httpClient.Do(req) + if err != nil { + return deleted, err + } + + var result struct { + Records []struct { + URI string `json:"uri"` + } `json:"records"` + Cursor string `json:"cursor"` + } + + respBody, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return deleted, fmt.Errorf("list records failed: %s - %s", resp.Status, string(respBody)) + } + + if err := json.Unmarshal(respBody, &result); err != nil { + return deleted, err + } + + if len(result.Records) == 0 { + break + } + + for _, record := range result.Records { + parts := strings.Split(record.URI, "/") + if len(parts) < 2 { + continue + } + rkey := parts[len(parts)-1] + + if err := p.DeleteRecord(session, "app.bsky.feed.post", rkey); err != nil { + continue + } + deleted++ + } + + cursor = result.Cursor + if cursor == "" { + break + } + } + + return deleted, nil +} + +// DeleteRecord deletes a single record from an account +func (p *Publisher) DeleteRecord(session *PDSSession, collection, rkey string) error { + payload := map[string]interface{}{ + "repo": session.DID, + "collection": collection, + "rkey": rkey, + } + + body, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.deleteRecord", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + resp, err := p.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("delete record failed: %s - %s", resp.Status, string(respBody)) + } + + return nil +} + +// DeleteAccount deletes an account using PDS admin API +func (p *Publisher) DeleteAccount(adminPassword, did string) error { + payload := map[string]interface{}{ + "did": did, + } + + body, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.admin.deleteAccount", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth("admin", adminPassword) + + resp, err := p.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("delete account failed: %s - %s", resp.Status, string(respBody)) + } + + return nil +} + +// TakedownAccount applies a takedown to an account +func (p *Publisher) TakedownAccount(adminPassword, did, reason string) error { + payload := map[string]interface{}{ + "subject": map[string]interface{}{ + "$type": "com.atproto.admin.defs#repoRef", + "did": did, + }, + "takedown": map[string]interface{}{ + "applied": true, + "ref": reason, + }, + } + + body, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.admin.updateSubjectStatus", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth("admin", adminPassword) + + resp, err := p.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("takedown account failed: %s - %s", resp.Status, string(respBody)) + } + + return nil +} + +// RestoreAccount removes a takedown from an account +func (p *Publisher) RestoreAccount(adminPassword, did string) error { + payload := map[string]interface{}{ + "subject": map[string]interface{}{ + "$type": "com.atproto.admin.defs#repoRef", + "did": did, + }, + "takedown": map[string]interface{}{ + "applied": false, + }, + } + + body, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.admin.updateSubjectStatus", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth("admin", adminPassword) + + resp, err := p.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("restore account failed: %s - %s", resp.Status, string(respBody)) + } + + return nil +} diff --git a/publisher b/publisher new file mode 100755 index 0000000..fd80ab9 Binary files /dev/null and b/publisher differ diff --git a/publisher.go b/publisher.go new file mode 100644 index 0000000..7b630ac --- /dev/null +++ b/publisher.go @@ -0,0 +1,395 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/1440news/commons" +) + +// Publisher handles posting items to AT Protocol PDS +type Publisher struct { + pdsHost string + httpClient *http.Client +} + +// PDSSession holds authentication info for a PDS account +type PDSSession struct { + DID string `json:"did"` + Handle string `json:"handle"` + AccessJwt string `json:"accessJwt"` + RefreshJwt string `json:"refreshJwt"` +} + +// BskyPost represents an app.bsky.feed.post record +type BskyPost struct { + Type string `json:"$type"` + Text string `json:"text"` + CreatedAt string `json:"createdAt"` + Facets []BskyFacet `json:"facets,omitempty"` + Embed *BskyEmbed `json:"embed,omitempty"` +} + +type BskyFacet struct { + Index BskyByteSlice `json:"index"` + Features []BskyFeature `json:"features"` +} + +type BskyByteSlice struct { + ByteStart int `json:"byteStart"` + ByteEnd int `json:"byteEnd"` +} + +type BskyFeature struct { + Type string `json:"$type"` + URI string `json:"uri,omitempty"` + Tag string `json:"tag,omitempty"` +} + +type BskyEmbed struct { + Type string `json:"$type"` + External *BskyExternal `json:"external,omitempty"` + Images []BskyImage `json:"images,omitempty"` +} + +type BskyExternal struct { + URI string `json:"uri"` + Title string `json:"title"` + Description string `json:"description"` + Thumb *BlobRef `json:"thumb,omitempty"` +} + +type BskyImage struct { + Alt string `json:"alt"` + Image *BlobRef `json:"image"` + AspectRatio *BskyAspectRatio `json:"aspectRatio,omitempty"` +} + +type BskyAspectRatio struct { + Width int `json:"width"` + Height int `json:"height"` +} + +// NewPublisher creates a new Publisher instance +func NewPublisher(pdsHost string) *Publisher { + return &Publisher{ + pdsHost: pdsHost, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +// TID alphabet for base32-sortable encoding +const tidAlphabet = "234567abcdefghijklmnopqrstuvwxyz" + +// GenerateRkey creates a deterministic TID-format rkey from a GUID and timestamp +func GenerateRkey(guid string, timestamp time.Time) string { + if guid == "" { + return "" + } + + microsInt := timestamp.UnixMicro() + if microsInt < 0 { + microsInt = 0 + } + micros := uint64(microsInt) & ((1 << 53) - 1) + + hash := sha256.Sum256([]byte(guid)) + h1 := uint64(hash[0]) ^ uint64(hash[2]) ^ uint64(hash[4]) ^ uint64(hash[6]) + h2 := uint64(hash[1]) ^ uint64(hash[3]) ^ uint64(hash[5]) ^ uint64(hash[7]) + clockID := (h1 << 2) | (h2 >> 6) + clockID = clockID & ((1 << 10) - 1) + + tid := (micros << 10) | clockID + + var result [13]byte + for i := 12; i >= 0; i-- { + result[i] = tidAlphabet[tid&0x1f] + tid >>= 5 + } + + return string(result[:]) +} + +// extractURLs finds all URLs in a string +func extractURLs(text string) []string { + urlRegex := regexp.MustCompile(`https?://[^\s<>"'\)]+`) + matches := urlRegex.FindAllString(text, -1) + + var urls []string + for _, u := range matches { + u = strings.TrimRight(u, ".,;:!?") + if u != "" { + urls = append(urls, u) + } + } + return urls +} + +// toCamelCaseTag converts a tag string to camelCase hashtag format +func toCamelCaseTag(tag string) string { + tag = strings.TrimSpace(tag) + if tag == "" { + return "" + } + + tag = strings.TrimPrefix(tag, "#") + + words := strings.FieldsFunc(tag, func(r rune) bool { + return r == ' ' || r == '-' || r == '_' + }) + + if len(words) == 0 { + return "" + } + + if len(words) == 1 { + return strings.ToLower(words[0]) + } + + var result strings.Builder + for i, word := range words { + if word == "" { + continue + } + runes := []rune(word) + if len(runes) > 0 { + if i == 0 || result.Len() == 0 { + result.WriteString(strings.ToLower(word)) + } else { + result.WriteString(strings.ToUpper(string(runes[0]))) + if len(runes) > 1 { + result.WriteString(strings.ToLower(string(runes[1:]))) + } + } + } + } + return result.String() +} + +// formatTagsForPost converts item tags to hashtag text and facets +func formatTagsForPost(tags []string, textOffset int) (string, []BskyFacet) { + if len(tags) == 0 { + return "", nil + } + + seen := make(map[string]bool) + var hashtags []string + for _, tag := range tags { + camel := toCamelCaseTag(tag) + if camel == "" || seen[strings.ToLower(camel)] { + continue + } + seen[strings.ToLower(camel)] = true + hashtags = append(hashtags, camel) + } + + if len(hashtags) == 0 { + return "", nil + } + + if len(hashtags) > 5 { + hashtags = hashtags[:5] + } + + var line strings.Builder + var facets []BskyFacet + currentOffset := textOffset + + for i, ht := range hashtags { + if i > 0 { + line.WriteString(" ") + currentOffset++ + } + + hashtagText := "#" + ht + byteStart := currentOffset + byteEnd := currentOffset + len(hashtagText) + + line.WriteString(hashtagText) + + facets = append(facets, BskyFacet{ + Index: BskyByteSlice{ + ByteStart: byteStart, + ByteEnd: byteEnd, + }, + Features: []BskyFeature{{ + Type: "app.bsky.richtext.facet#tag", + Tag: ht, + }}, + }) + + currentOffset = byteEnd + } + + return line.String(), facets +} + +// PublishItem posts a feed item to the PDS +func (p *Publisher) PublishItem(session *PDSSession, item *shared.Item) (string, error) { + if item.GUID == "" && item.Link == "" { + return "", fmt.Errorf("item has no GUID or link, cannot publish") + } + + urlSet := make(map[string]bool) + var allURLs []string + + if item.Link != "" { + urlSet[item.Link] = true + allURLs = append(allURLs, item.Link) + } + + descURLs := extractURLs(item.Description) + for _, u := range descURLs { + if strings.Contains(u, "news.ycombinator.com/item") && !urlSet[u] { + urlSet[u] = true + allURLs = append(allURLs, u) + break + } + } + + if len(allURLs) < 2 && item.Enclosure != nil && item.Enclosure.URL != "" { + encType := strings.ToLower(item.Enclosure.Type) + if strings.HasPrefix(encType, "audio/") || strings.HasPrefix(encType, "video/") { + if !urlSet[item.Enclosure.URL] { + currentURLLen := 0 + for _, u := range allURLs { + currentURLLen += len(u) + 2 + } + enclosureLen := len(item.Enclosure.URL) + 2 + if currentURLLen+enclosureLen < 235 { + urlSet[item.Enclosure.URL] = true + allURLs = append(allURLs, item.Enclosure.URL) + } + } + } + } + + primaryURL := "" + if len(allURLs) > 0 { + primaryURL = allURLs[0] + } + + createdAt := time.Now() + if !item.PubDate.IsZero() { + createdAt = item.PubDate + } + + postText := "" + var facets []BskyFacet + + if len(item.Tags) > 0 { + tagLine, tagFacets := formatTagsForPost(item.Tags, 0) + postText = tagLine + facets = tagFacets + } + + post := BskyPost{ + Type: "app.bsky.feed.post", + Text: postText, + CreatedAt: createdAt.Format(time.RFC3339), + Facets: facets, + } + + if primaryURL != "" { + external := &BskyExternal{ + URI: primaryURL, + Title: item.Title, + Description: truncate(stripHTML(item.Description), 300), + } + + if len(item.ImageURLs) > 0 { + if thumb := p.fetchAndUploadImage(session, item.ImageURLs[0]); thumb != nil { + external.Thumb = thumb + } + } + + post.Embed = &BskyEmbed{ + Type: "app.bsky.embed.external", + External: external, + } + } + + guidForRkey := item.GUID + if guidForRkey == "" { + guidForRkey = item.Link + } + rkeyTime := item.PubDate + if rkeyTime.IsZero() { + rkeyTime = item.DiscoveredAt + } + rkey := GenerateRkey(guidForRkey, rkeyTime) + + payload := map[string]interface{}{ + "repo": session.DID, + "collection": "app.bsky.feed.post", + "rkey": rkey, + "record": post, + } + + body, err := json.Marshal(payload) + if err != nil { + return "", err + } + + req, err := http.NewRequest("POST", p.pdsHost+"/xrpc/com.atproto.repo.createRecord", bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+session.AccessJwt) + + resp, err := p.httpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("create record failed: %s - %s", resp.Status, string(respBody)) + } + + var result struct { + URI string `json:"uri"` + CID string `json:"cid"` + } + if err := json.Unmarshal(respBody, &result); err != nil { + return "", err + } + + return result.URI, nil +} + +func truncate(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen-3] + "..." +} + +func stripHTML(s string) string { + tagRegex := regexp.MustCompile(`<[^>]*>`) + s = tagRegex.ReplaceAllString(s, "") + + s = strings.ReplaceAll(s, "&", "&") + s = strings.ReplaceAll(s, "<", "<") + s = strings.ReplaceAll(s, ">", ">") + s = strings.ReplaceAll(s, """, "\"") + s = strings.ReplaceAll(s, "'", "'") + s = strings.ReplaceAll(s, " ", " ") + + spaceRegex := regexp.MustCompile(`\s+`) + s = spaceRegex.ReplaceAllString(s, " ") + + return strings.TrimSpace(s) +} diff --git a/server.go b/server.go new file mode 100644 index 0000000..f2f4a33 --- /dev/null +++ b/server.go @@ -0,0 +1,726 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + + "github.com/1440news/commons" +) + +// StartServer starts the HTTP server for the publisher API +func (s *PublisherService) StartServer(addr string) error { + mux := http.NewServeMux() + + // Health check + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Publishing APIs + mux.HandleFunc("/api/enablePublish", s.handleEnablePublish) + mux.HandleFunc("/api/disablePublish", s.handleDisablePublish) + mux.HandleFunc("/api/setPublishStatus", s.handleSetPublishStatus) + mux.HandleFunc("/api/publishEnabled", s.handlePublishEnabled) + mux.HandleFunc("/api/publishDenied", s.handlePublishDenied) + mux.HandleFunc("/api/publishCandidates", s.handlePublishCandidates) + mux.HandleFunc("/api/unpublishedItems", s.handleUnpublishedItems) + mux.HandleFunc("/api/testPublish", s.handleTestPublish) + mux.HandleFunc("/api/publishFeed", s.handlePublishFeed) + mux.HandleFunc("/api/publishFeedFull", s.handlePublishFeedFull) + mux.HandleFunc("/api/createAccount", s.handleCreateAccount) + mux.HandleFunc("/api/updateProfile", s.handleUpdateProfile) + mux.HandleFunc("/api/deriveHandle", s.handleDeriveHandle) + mux.HandleFunc("/api/resetAllPublishing", s.handleResetAllPublishing) + mux.HandleFunc("/api/refreshProfiles", s.handleRefreshProfiles) + + fmt.Printf("Publisher API running at http://%s\n", addr) + return http.ListenAndServe(addr, mux) +} + +func (s *PublisherService) handleEnablePublish(w http.ResponseWriter, r *http.Request) { + feedURL := r.URL.Query().Get("url") + account := r.URL.Query().Get("account") + if feedURL == "" { + http.Error(w, "url parameter required", http.StatusBadRequest) + return + } + + feedURL = shared.NormalizeURL(feedURL) + + if account == "" { + account = shared.DeriveHandleFromFeed(feedURL) + if account == "" { + http.Error(w, "could not derive account handle from URL", http.StatusBadRequest) + return + } + } + + if err := s.SetPublishStatus(feedURL, "pass", account); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + count, _ := s.GetUnpublishedItemCount(feedURL) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "pass", + "url": feedURL, + "account": account, + "unpublished_items": count, + }) +} + +func (s *PublisherService) handleDisablePublish(w http.ResponseWriter, r *http.Request) { + feedURL := r.URL.Query().Get("url") + if feedURL == "" { + http.Error(w, "url parameter required", http.StatusBadRequest) + return + } + + feedURL = shared.NormalizeURL(feedURL) + + if err := s.SetPublishStatus(feedURL, "skip", ""); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "skip", + "url": feedURL, + }) +} + +func (s *PublisherService) handleSetPublishStatus(w http.ResponseWriter, r *http.Request) { + feedURL := r.URL.Query().Get("url") + status := r.URL.Query().Get("status") + account := r.URL.Query().Get("account") + + if feedURL == "" { + http.Error(w, "url parameter required", http.StatusBadRequest) + return + } + if status != "pass" && status != "skip" && status != "hold" && status != "drop" { + http.Error(w, "status must be 'pass', 'hold', 'skip', or 'drop'", http.StatusBadRequest) + return + } + + feedURL = shared.NormalizeURL(feedURL) + + result := map[string]interface{}{ + "url": feedURL, + "status": status, + } + + if status == "pass" { + if account == "" { + account = shared.DeriveHandleFromFeed(feedURL) + } + result["account"] = account + } + + if err := s.SetPublishStatus(feedURL, status, account); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(result) +} + +func (s *PublisherService) handlePublishEnabled(w http.ResponseWriter, r *http.Request) { + feeds, err := s.GetFeedsByPublishStatus("pass") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + type FeedPublishInfo struct { + URL string `json:"url"` + Title string `json:"title"` + Account string `json:"account"` + UnpublishedCount int `json:"unpublished_count"` + } + + var result []FeedPublishInfo + for _, f := range feeds { + count, _ := s.GetUnpublishedItemCount(f.URL) + result = append(result, FeedPublishInfo{ + URL: f.URL, + Title: f.Title, + Account: f.PublishAccount, + UnpublishedCount: count, + }) + } + + if result == nil { + result = []FeedPublishInfo{} + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(result) +} + +func (s *PublisherService) handlePublishDenied(w http.ResponseWriter, r *http.Request) { + feeds, err := s.GetFeedsByPublishStatus("skip") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + type FeedDeniedInfo struct { + URL string `json:"url"` + Title string `json:"title"` + SourceHost string `json:"source_host"` + } + + var result []FeedDeniedInfo + for _, f := range feeds { + result = append(result, FeedDeniedInfo{ + URL: f.URL, + Title: f.Title, + SourceHost: f.DomainHost, + }) + } + + if result == nil { + result = []FeedDeniedInfo{} + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(result) +} + +func (s *PublisherService) handlePublishCandidates(w http.ResponseWriter, r *http.Request) { + limit := 50 + if l := r.URL.Query().Get("limit"); l != "" { + fmt.Sscanf(l, "%d", &limit) + if limit > 200 { + limit = 200 + } + } + + feeds, err := s.GetPublishCandidates(limit) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + type CandidateInfo struct { + URL string `json:"url"` + Title string `json:"title"` + Category string `json:"category"` + SourceHost string `json:"source_host"` + ItemCount int `json:"item_count"` + DerivedHandle string `json:"derived_handle"` + } + + var result []CandidateInfo + for _, f := range feeds { + result = append(result, CandidateInfo{ + URL: f.URL, + Title: f.Title, + Category: f.Category, + SourceHost: f.DomainHost, + ItemCount: f.ItemCount, + DerivedHandle: shared.DeriveHandleFromFeed(f.URL), + }) + } + + if result == nil { + result = []CandidateInfo{} + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(result) +} + +func (s *PublisherService) handleUnpublishedItems(w http.ResponseWriter, r *http.Request) { + feedURL := r.URL.Query().Get("url") + if feedURL == "" { + http.Error(w, "url parameter required", http.StatusBadRequest) + return + } + + limit := 50 + if l := r.URL.Query().Get("limit"); l != "" { + fmt.Sscanf(l, "%d", &limit) + if limit > 200 { + limit = 200 + } + } + + items, err := s.GetUnpublishedItems(feedURL, limit) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if items == nil { + items = []*shared.Item{} + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(items) +} + +func (s *PublisherService) handleTestPublish(w http.ResponseWriter, r *http.Request) { + feedURL := r.URL.Query().Get("feedUrl") + guidParam := r.URL.Query().Get("guid") + handle := r.URL.Query().Get("handle") + password := r.URL.Query().Get("password") + pdsHost := r.URL.Query().Get("pds") + + if feedURL == "" || guidParam == "" { + http.Error(w, "feedUrl and guid parameters required", http.StatusBadRequest) + return + } + if handle == "" || password == "" { + http.Error(w, "handle and password parameters required", http.StatusBadRequest) + return + } + if pdsHost == "" { + pdsHost = s.pdsHost + } + + item, err := s.GetItemByGUID(feedURL, guidParam) + if err != nil { + http.Error(w, "item not found: "+err.Error(), http.StatusNotFound) + return + } + + publisher := NewPublisher(pdsHost) + session, err := publisher.CreateSession(handle, password) + if err != nil { + http.Error(w, "auth failed: "+err.Error(), http.StatusUnauthorized) + return + } + + uri, err := publisher.PublishItem(session, item) + if err != nil { + http.Error(w, "publish failed: "+err.Error(), http.StatusInternalServerError) + return + } + + s.MarkItemPublished(item.FeedURL, item.GUID, uri) + + rkeyTime := item.PubDate + if rkeyTime.IsZero() { + rkeyTime = item.DiscoveredAt + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "published", + "uri": uri, + "feedUrl": item.FeedURL, + "guid": item.GUID, + "title": item.Title, + "rkey": GenerateRkey(item.GUID, rkeyTime), + }) +} + +func (s *PublisherService) handlePublishFeed(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "not_implemented", + }) +} + +func (s *PublisherService) handlePublishFeedFull(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "not_implemented", + }) +} + +func (s *PublisherService) handleCreateAccount(w http.ResponseWriter, r *http.Request) { + handle := r.URL.Query().Get("handle") + email := r.URL.Query().Get("email") + password := r.URL.Query().Get("password") + pdsHost := r.URL.Query().Get("pds") + inviteCode := r.URL.Query().Get("inviteCode") + pdsAdminPassword := r.URL.Query().Get("pdsAdminPassword") + + if handle == "" || password == "" { + http.Error(w, "handle and password parameters required", http.StatusBadRequest) + return + } + if pdsHost == "" { + pdsHost = s.pdsHost + } + if email == "" { + email = handle + "@1440.news" + } + + publisher := NewPublisher(pdsHost) + + if pdsAdminPassword != "" && inviteCode == "" { + code, err := publisher.CreateInviteCode(pdsAdminPassword, 1) + if err != nil { + http.Error(w, "create invite failed: "+err.Error(), http.StatusInternalServerError) + return + } + inviteCode = code + } + + session, err := publisher.CreateAccount(handle, email, password, inviteCode) + if err != nil { + http.Error(w, "create account failed: "+err.Error(), http.StatusInternalServerError) + return + } + + if err := publisher.FollowAsDirectory(session.DID); err != nil { + fmt.Printf("API: directory follow failed for %s: %v\n", handle, err) + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "created", + "handle": session.Handle, + "did": session.DID, + }) +} + +func (s *PublisherService) handleUpdateProfile(w http.ResponseWriter, r *http.Request) { + handle := r.URL.Query().Get("handle") + password := r.URL.Query().Get("password") + pdsHost := r.URL.Query().Get("pds") + displayName := r.URL.Query().Get("displayName") + description := r.URL.Query().Get("description") + faviconURL := r.URL.Query().Get("faviconUrl") + + if handle == "" || password == "" { + http.Error(w, "handle and password parameters required", http.StatusBadRequest) + return + } + if pdsHost == "" { + pdsHost = s.pdsHost + } + + publisher := NewPublisher(pdsHost) + + session, err := publisher.CreateSession(handle, password) + if err != nil { + http.Error(w, "auth failed: "+err.Error(), http.StatusUnauthorized) + return + } + + var avatar *BlobRef + if faviconURL != "" { + faviconData, mimeType, err := FetchFaviconBytes(faviconURL) + if err != nil { + http.Error(w, "fetch favicon failed: "+err.Error(), http.StatusBadRequest) + return + } + avatar, err = publisher.UploadBlob(session, faviconData, mimeType) + if err != nil { + http.Error(w, "upload favicon failed: "+err.Error(), http.StatusInternalServerError) + return + } + } + + if err := publisher.UpdateProfile(session, displayName, description, avatar); err != nil { + http.Error(w, "update profile failed: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "updated", + "handle": handle, + "displayName": displayName, + "hasAvatar": avatar != nil, + }) +} + +func (s *PublisherService) handleDeriveHandle(w http.ResponseWriter, r *http.Request) { + feedURL := r.URL.Query().Get("url") + if feedURL == "" { + http.Error(w, "url parameter required", http.StatusBadRequest) + return + } + + handle := shared.DeriveHandleFromFeed(feedURL) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "url": feedURL, + "handle": handle, + }) +} + +func (s *PublisherService) handleResetAllPublishing(w http.ResponseWriter, r *http.Request) { + accountsCleared, err := s.db.Exec(`UPDATE feeds SET publish_account = NULL WHERE publish_account IS NOT NULL`) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + itemsCleared, err := s.db.Exec(`UPDATE items SET published_at = NULL WHERE published_at IS NOT NULL`) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + statusReset, err := s.db.Exec(`UPDATE feeds SET publish_status = 'hold' WHERE publish_status IS NOT NULL`) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "success": true, + "accounts_cleared": accountsCleared, + "items_cleared": itemsCleared, + "status_reset": statusReset, + }) +} + +func (s *PublisherService) handleRefreshProfiles(w http.ResponseWriter, r *http.Request) { + password := r.URL.Query().Get("password") + pdsHost := r.URL.Query().Get("pds") + + if password == "" { + http.Error(w, "password parameter required", http.StatusBadRequest) + return + } + if pdsHost == "" { + pdsHost = s.pdsHost + } + + publisher := NewPublisher(pdsHost) + s.RefreshAllProfiles(publisher, password) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "success": true, + "message": "profiles refreshed", + }) +} + +// Database helper methods + +func (s *PublisherService) SetPublishStatus(feedURL, status, account string) error { + feedURL = shared.NormalizeURL(feedURL) + + if status == "pass" && account == "" { + account = shared.DeriveHandleFromFeed(feedURL) + } + + _, err := s.db.Exec(` + UPDATE feeds SET publish_status = $1, publish_account = $2 WHERE url = $3 + `, status, shared.NullableString(account), feedURL) + return err +} + +func (s *PublisherService) GetFeedsByPublishStatus(status string) ([]*shared.Feed, error) { + rows, err := s.db.Query(` + SELECT url, type, category, title, description, language, site_url, + discovered_at, last_checked_at, next_check_at, last_build_date, + etag, last_modified, + status, last_error, last_error_at, + source_url, domain_host, domain_tld, + item_count, oldest_item_date, newest_item_date, + no_update, + publish_status, publish_account + FROM feeds + WHERE publish_status = $1 + `, status) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanFeeds(rows) +} + +func (s *PublisherService) GetPublishCandidates(limit int) ([]*shared.Feed, error) { + rows, err := s.db.Query(` + SELECT url, type, category, title, description, language, site_url, + discovered_at, last_checked_at, next_check_at, last_build_date, + etag, last_modified, + status, last_error, last_error_at, + source_url, domain_host, domain_tld, + item_count, oldest_item_date, newest_item_date, + no_update, + publish_status, publish_account + FROM feeds + WHERE publish_status = 'hold' AND item_count > 0 AND status = 'pass' + ORDER BY item_count DESC + LIMIT $1 + `, limit) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanFeeds(rows) +} + +func (s *PublisherService) GetUnpublishedItemCount(feedURL string) (int, error) { + var count int + err := s.db.QueryRow(` + SELECT COUNT(*) FROM items WHERE feed_url = $1 AND published_at IS NULL + `, feedURL).Scan(&count) + return count, err +} + +func (s *PublisherService) GetItemByGUID(feedURL, guid string) (*shared.Item, error) { + items, err := s.db.Query(` + SELECT feed_url, guid, title, link, description, content, author, pub_date, discovered_at, updated_at, + enclosure_url, enclosure_type, enclosure_length, image_urls, tags, + published_at, published_uri + FROM items + WHERE feed_url = $1 AND guid = $2 + `, feedURL, guid) + if err != nil { + return nil, err + } + defer items.Close() + + result, err := scanItems(items) + if err != nil { + return nil, err + } + if len(result) == 0 { + return nil, fmt.Errorf("item not found") + } + return result[0], nil +} + +func (s *PublisherService) RefreshAllProfiles(publisher *Publisher, feedPassword string) { + rows, err := s.db.Query(` + SELECT url, title, description, site_url, domain_host as source_host, publish_account + FROM feeds + WHERE publish_account IS NOT NULL AND publish_account <> '' + `) + if err != nil { + fmt.Printf("RefreshProfiles: query error: %v\n", err) + return + } + defer rows.Close() + + for rows.Next() { + var feedURL, account string + var title, description, siteURL, sourceHost *string + if err := rows.Scan(&feedURL, &title, &description, &siteURL, &sourceHost, &account); err != nil { + continue + } + + session, err := publisher.CreateSession(account, feedPassword) + if err != nil { + fmt.Printf("RefreshProfiles: login failed for %s: %v\n", account, err) + continue + } + + displayName := shared.StringValue(title) + if displayName == "" { + displayName = account + } + desc := stripHTML(shared.StringValue(description)) + if desc == "" { + desc = "News feed via 1440.news" + } + feedURLFull := "https://" + feedURL + desc = feedURLFull + "\n\n" + desc + + if len(displayName) > 64 { + displayName = displayName[:61] + "..." + } + if len(desc) > 256 { + desc = desc[:253] + "..." + } + + var avatar *BlobRef + faviconSource := shared.StringValue(siteURL) + if faviconSource == "" { + faviconSource = shared.StringValue(sourceHost) + } + if faviconSource != "" { + faviconData, mimeType, err := FetchFaviconBytes(faviconSource) + if err == nil && len(faviconData) > 0 { + avatar, _ = publisher.UploadBlob(session, faviconData, mimeType) + } + } + + if err := publisher.UpdateProfile(session, displayName, desc, avatar); err != nil { + fmt.Printf("RefreshProfiles: update failed for %s: %v\n", account, err) + } else { + fmt.Printf("RefreshProfiles: updated %s\n", account) + } + } +} + +// scanFeeds helper +func scanFeeds(rows interface{ Next() bool; Scan(...interface{}) error; Err() error }) ([]*shared.Feed, error) { + var feeds []*shared.Feed + + for rows.Next() { + feed := &shared.Feed{} + var feedType, category, title, description, language, siteURL *string + var lastCheckedAt, nextCheckAt, lastBuildDate *interface{} + var etag, lastModified, lastError, sourceURL, domainTLD *string + var lastErrorAt *interface{} + var oldestItemDate, newestItemDate *interface{} + var publishStatus, publishAccount *string + + err := rows.Scan( + &feed.URL, &feedType, &category, &title, &description, &language, &siteURL, + &feed.DiscoveredAt, &lastCheckedAt, &nextCheckAt, &lastBuildDate, + &etag, &lastModified, + &feed.Status, &lastError, &lastErrorAt, + &sourceURL, &feed.DomainHost, &domainTLD, + &feed.ItemCount, &oldestItemDate, &newestItemDate, + &feed.NoUpdate, + &publishStatus, &publishAccount, + ) + if err != nil { + continue + } + + feed.Type = shared.StringValue(feedType) + feed.Category = shared.StringValue(category) + feed.Title = shared.StringValue(title) + feed.Description = shared.StringValue(description) + feed.Language = shared.StringValue(language) + feed.SiteURL = shared.StringValue(siteURL) + feed.ETag = shared.StringValue(etag) + feed.LastModified = shared.StringValue(lastModified) + feed.LastError = shared.StringValue(lastError) + feed.SourceURL = shared.StringValue(sourceURL) + feed.DomainTLD = shared.StringValue(domainTLD) + feed.PublishStatus = shared.StringValue(publishStatus) + feed.PublishAccount = shared.StringValue(publishAccount) + + feeds = append(feeds, feed) + } + + return feeds, rows.Err() +} + +func init() { + // Try to load pds.env if it exists + if data, err := os.ReadFile("pds.env"); err == nil { + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + if os.Getenv(key) == "" { + os.Setenv(key, value) + } + } + } + } +}