first commit

Dockerfile

@@ -0,0 +1,10 @@
+
+FROM debian:buster-slim
+# Copy the binary to the production image from the builder stage.
+WORKDIR /app
+
+COPY ./main /app/main
+# Set workdir.
+RUN chmod +x /app/main
+# Run the web service on container startup.
+CMD ["./main"]

bulid.bat

@@ -0,0 +1,4 @@
+set GOOS=linux
+set GOARCH=amd64
+
+go build main.go

common/log/log.go

@@ -0,0 +1,196 @@
+package log
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	rotatelogs "github.com/lestrrat-go/file-rotatelogs"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	logrus.SetFormatter(&DFormatter{})
+	logrus.SetLevel(DebugLevel)
+	logrus.SetOutput(os.Stdout)
+}
+
+// type PanicLevel = logrus.PanicLevel
+const (
+	PanicLevel logrus.Level = logrus.PanicLevel
+	FatalLevel logrus.Level = logrus.FatalLevel
+	ErrorLevel logrus.Level = logrus.ErrorLevel
+	WarnLevel  logrus.Level = logrus.WarnLevel
+	InfoLevel  logrus.Level = logrus.InfoLevel
+	DebugLevel logrus.Level = logrus.DebugLevel
+)
+
+var (
+	fileConn *rotatelogs.RotateLogs
+)
+
+type LogConfig struct {
+	Filename        string        // 日志文件
+	RetainFileCount uint          // 保留日志文件数量
+	IsDevelop       bool          // 开发模式会输出日志到标准输出
+	FileSplitTime   time.Duration // 日志文件拆分时间间隔 time.Hour*24
+}
+
+func WithField(key string, value string) *logrus.Entry {
+	return logrus.WithField(key, value)
+}
+
+// config need to be correct JSON as string: {"filename":"default.log","maxsize":100}
+func SetLogger(config LogConfig) *logrus.Logger {
+	/* 日志轮转相关函数
+	   `WithLinkName` 为最新的日志建立软连接
+	   `WithRotationTime` 设置日志分割的时间，隔多久分割一次
+	   WithMaxAge 和 WithRotationCount二者只能设置一个
+	     `WithMaxAge` 设置文件清理前的最长保存时间
+	     `WithRotationCount` 设置文件清理前最多保存的个数
+	*/
+	//fileName := config.Filename + ".%Y%m%d"
+	fileName := ""
+	if config.FileSplitTime > time.Hour {
+		fileName = fmt.Sprintf(config.Filename, "%Y%m%d")
+	} else if config.FileSplitTime > time.Minute*30 {
+		fileName = fmt.Sprintf(config.Filename, "%Y%m%d_%H")
+	} else {
+		fileName = fmt.Sprintf(config.Filename, "%Y%m%d_%H_%M")
+	}
+	fileConn, _ := rotatelogs.New(
+		fileName,
+		//rotatelogs.WithLinkName(config.Filename),
+		rotatelogs.WithRotationCount(config.RetainFileCount),
+		rotatelogs.WithRotationTime(config.FileSplitTime),
+	)
+	if config.IsDevelop {
+		fmt.Println("isDeubg=true; log out console")
+		logrus.SetLevel(DebugLevel)
+		logrus.SetOutput(os.Stdout)
+	} else {
+		fmt.Println("isDeubg=false; log out file")
+		logrus.SetOutput(fileConn)
+	}
+	return logrus.StandardLogger()
+}
+func GetFileConn() *rotatelogs.RotateLogs {
+	return fileConn
+}
+func SetLevel(logLevel logrus.Level) {
+	logrus.SetLevel(logLevel)
+}
+
+type DFormatter struct {
+	TimestampFormat string
+}
+
+func (f *DFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	timestampFormat := f.TimestampFormat
+	if timestampFormat == "" {
+		timestampFormat = "2006/01/02 15:04:05"
+	}
+
+	_, file, line, ok := runtime.Caller(9)
+	if !ok {
+		file = "???"
+		line = 0
+	}
+	file = filepath.Base(file) // 只取文件路径的文件名
+	// _, filename := path.Split(file)
+	msg := entry.Time.Format(timestampFormat) +
+		" " + strings.ToUpper(entry.Level.String()) +
+		" [" + file + ":" + strconv.Itoa(line) + "] " +
+		entry.Message + "\n"
+
+	return []byte(msg), nil
+}
+
+func Debugf(format string, args ...interface{}) {
+	logrus.Debugf(format, args...)
+}
+
+func Infof(format string, args ...interface{}) {
+	logrus.Infof(format, args...)
+}
+
+func Warnf(format string, args ...interface{}) {
+	logrus.Warnf(format, args...)
+}
+
+func Errorf(format string, args ...interface{}) {
+	logrus.Errorf(format, args...)
+}
+
+func Fatalf(format string, args ...interface{}) {
+	logrus.Fatalf(format, args...)
+}
+
+func Panicf(format string, args ...interface{}) {
+	logrus.Panicf(format, args...)
+}
+
+// 以下函数不可以写成这种形式
+// func Debug(args ...interface{}) {
+// 	logrus.Debug(args...)
+// }
+
+func Debug(args ...interface{}) {
+	debug(args...)
+}
+
+func debug(args ...interface{}) {
+	logrus.Debug(args...)
+}
+
+func Println(args ...interface{}) {
+	println(args...)
+}
+
+func println(args ...interface{}) {
+	logrus.Println(args...)
+}
+
+func Info(args ...interface{}) {
+	info(args...)
+}
+
+func info(args ...interface{}) {
+	logrus.Info(args...)
+}
+
+func Warn(args ...interface{}) {
+	warn(args...)
+}
+
+func warn(args ...interface{}) {
+	logrus.Warn(args...)
+}
+
+func Error(args ...interface{}) {
+	ferror(args...)
+}
+
+func ferror(args ...interface{}) {
+	logrus.Error(args...)
+}
+
+func Fatal(args ...interface{}) {
+	fatal(args...)
+}
+
+func fatal(args ...interface{}) {
+	logrus.Fatal(args...)
+}
+
+func Panic(args ...interface{}) {
+	panic(args...)
+}
+
+func panic(args ...interface{}) {
+	logrus.Panic(args...)
+}

consts/status.go

@@ -0,0 +1,8 @@
+package consts
+
+const (
+	STATUS_INIT   = 0
+	STATUS_SPIDER = 1
+	STATUS_BRIEF  = 2
+	STATUS_IMG    = 3
+)

db/gorm.go

@@ -0,0 +1,18 @@
+package db
+
+import (
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+)
+
+var Gorm *gorm.DB
+
+func init() {
+	// dsn := "root:123456@tcp(localhost:3306)/rental?charset=utf8"
+	dsn := "homapp:PaqvTcSXUzNc9M9@tcp(34.105.87.213:3306)/homapp?charset=utf8"
+	var err error
+	Gorm, err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		panic(err)
+	}
+}

dom.json

@@ -0,0 +1,15 @@
+{
+    "www.cbc.ca":	"#detailContent",
+    "financialpost.com":	".article-content-story",
+    "www.lapresse.ca":	".mainStory",
+    "www.vernonmorningstar.com":	"#details-body",
+    "torontosun.com":	"#main-content",
+    "www.ledevoir.com":	"#articles",
+    "www.ctvnews.ca":	"#responsive_main",
+    "www.lesaffaires.com":	"#article_detail",
+    "www.theglobeandmail.com":	"#main-content",
+    "realestatemagazine.ca":	".entry-content",
+    "storeys.com":	".postpage-content",
+    "ici.radio-canada.ca":	".main-content",
+    "www.mpamag.com":	"article"
+}

go.mod

@@ -0,0 +1,24 @@
+module genBrief
+
+go 1.20
+
+require (
+	github.com/PuerkitoBio/goquery v1.9.1
+	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
+	github.com/sashabaranov/go-openai v1.20.4
+	github.com/sirupsen/logrus v1.9.3
+	gorm.io/driver/mysql v1.5.4
+	gorm.io/gorm v1.25.7
+)
+
+require (
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/jonboulle/clockwork v0.4.0 // indirect
+	github.com/lestrrat-go/strftime v1.0.6 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+)

go.sum

@@ -0,0 +1,80 @@
+github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
+github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
+github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
+github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8=
+github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
+github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4=
+github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
+github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ=
+github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sashabaranov/go-openai v1.20.4 h1:095xQ/fAtRa0+Rj21sezVJABgKfGPNbyx/sAN/hJUmg=
+github.com/sashabaranov/go-openai v1.20.4/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.4 h1:igQmHfKcbaTVyAIHNhhB888vvxh8EdQ2uSUT0LPcBso=
+gorm.io/driver/mysql v1.5.4/go.mod h1:9rYxJph/u9SWkWc9yY4XJ1F/+xO0S/ChOmbk3+Z5Tvs=
+gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=

main.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"genBrief/db"
+	"genBrief/model"
+	"genBrief/util"
+
+	"github.com/PuerkitoBio/goquery"
+	openai "github.com/sashabaranov/go-openai"
+)
+
+//go:embed dom.json
+var domJson string
+var domMap map[string]string
+
+func init() {
+	domMap = map[string]string{}
+	err := json.Unmarshal([]byte(domJson), &domMap)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func main() {
+	maxId := 4000
+	for {
+		newMaxId := loopMaxId(maxId)
+		if maxId == newMaxId {
+			return
+		}
+		maxId = newMaxId
+		loopMaxId(maxId)
+	}
+}
+
+func loopMaxId(maxId int) int {
+	var list []model.TNew
+	db.Gorm.Where("status='Pending'").Order("id asc").Limit(100).Find(&list)
+	for _, record := range list {
+		if len(record.Content) >= 300 {
+			record.Brief = genBrief(record.Content)
+		} else {
+			res, err := util.GetHtml(record.Url)
+			if err != nil {
+				fmt.Println("get Html false", err.Error())
+				record.Brief = ""
+			} else {
+				record.Content = getContent(res, "body")
+				if len(record.Content) >= 300 {
+					record.Brief = genBrief(record.Content)
+				} else {
+					record.Brief = ""
+					fmt.Println("skip", record.ID)
+				}
+			}
+		}
+		//
+		if record.Brief == "" {
+			record.Status = "Deactivated"
+		} else {
+			record.Status = "Activated"
+		}
+		fmt.Println("finish", record.ID, record.Brief, record.Status)
+		if err := db.Gorm.Save(&record).Error; err != nil {
+			fmt.Println("save", err.Error())
+		}
+	}
+	return maxId
+}
+
+func genBrief(content string) string {
+	client := openai.NewClient("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")
+	resp, err := client.CreateChatCompletion(
+		context.Background(),
+		openai.ChatCompletionRequest{
+			Model: openai.GPT3Dot5Turbo,
+			Messages: []openai.ChatCompletionMessage{
+				{
+					Role:    openai.ChatMessageRoleUser,
+					Content: content + "\r\n通过以上内容，生成中文概要，文字控制300字以内。",
+				},
+			},
+		},
+	)
+	if err != nil {
+		fmt.Printf("ChatCompletion error: %v\n", err)
+		return ""
+	}
+	return resp.Choices[0].Message.Content
+}
+
+func getContent(body []byte, dom string) string {
+	doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(body))
+	doc.Find("script").Remove()
+	doc.Find("noscript").Remove()
+	if dom == "" {
+		dom = "boby"
+	}
+	br := doc.Find(dom).Text()
+	if br == "" {
+		br = doc.Find("body").Text()
+	}
+	return util.TrimHtml(br)
+}

model/t_news.go

@@ -0,0 +1,14 @@
+package model
+
+type TNew struct {
+	ID      int    `json:"id" gorm:"primarykey"` // 主键ID
+	Url     string `json:"url"`
+	Brief   string `json:"brief"`
+	Status  string `json:"status"`
+	Content string `json:"content"`
+}
+
+// TableName DeviceSource 表名
+func (TNew) TableName() string {
+	return "t_news"
+}

model/t_news_body.go

@@ -0,0 +1,27 @@
+package model
+
+type TNewBody struct {
+	ID      int    `json:"id" gorm:"primarykey"` // 主键ID
+	Status  int    `json:"status"`
+	Brief   string `json:"brief"`
+	Url     string `json:"url"`
+	Content string `json:"content"`
+	Img     string `json:"img"`
+}
+
+// TableName DeviceSource 表名
+func (TNewBody) TableName() string {
+	return "t_news_body"
+}
+
+type TNewBodyHtml struct {
+	ID   int    `json:"id" gorm:"primarykey"` // 主键ID
+	Url  string `json:"url"`
+	Html string `json:"html"`
+	Img  string `json:"img"`
+}
+
+// TableName DeviceSource 表名
+func (TNewBodyHtml) TableName() string {
+	return "t_news_body"
+}

model/t_news_dom.go

@@ -0,0 +1,12 @@
+package model
+
+type TNewDom struct {
+	ID   uint   `json:"id" gorm:"primarykey"` // 主键ID
+	Host string `json:"host"`
+	Dom  string `json:"dom"`
+}
+
+// TableName DeviceSource 表名
+func (TNewDom) TableName() string {
+	return "t_news_dom"
+}

util/http.go

@@ -0,0 +1,24 @@
+package util
+
+import (
+	"io"
+	"net/http"
+)
+
+func GetHtml(url string) ([]byte, error) {
+	request, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	// 设置请求投
+	request.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
+	request.Header.Add("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
+	request.Header.Add("Connection", "keep-alive")
+	request.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36")
+	client := http.Client{}
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	return io.ReadAll(response.Body)
+}

util/str.go

@@ -0,0 +1,19 @@
+package util
+
+import "strings"
+
+func TrimHtml(data string) string {
+	nlist := strings.Split(data, "\n")
+	res := []string{}
+	for _, item := range nlist {
+		item = strings.Trim(item, " ")
+		if item == "" {
+			continue
+		} else if item == "\r" {
+			continue
+		} else {
+			res = append(res, item)
+		}
+	}
+	return strings.Join(res, "\n")
+}