add a lot

cli/ui/input.go

@@ -7,8 +7,11 @@ import (
 	"strings"
 )
 
+var (
+	scanner = bufio.NewScanner(os.Stdin)
+)
+
 func GetPath() (path string) {
-	scanner := bufio.NewScanner(os.Stdin)
 	for {
 		if scanner.Scan() {
 			input := scanner.Text()
@@ -18,18 +21,17 @@ func GetPath() (path string) {
 			)
 			stat, err := os.Stat(path)
 			if err != nil {
-				Printer("路径不存在！\n")
+				Output("路径不存在！\n")
 			} else if stat.IsDir() {
 				return
 			}
 		}
-		Printer("输入有误，请重新输入：")
+		Output("输入有误，请重新输入：")
 	}
 
 }
 
 func GetFile() (path string) {
-	scanner := bufio.NewScanner(os.Stdin)
 	for {
 		if scanner.Scan() {
 			input := scanner.Text()
@@ -39,17 +41,16 @@ func GetFile() (path string) {
 			)
 			stat, err := os.Stat(path)
 			if err != nil {
-				Printer("路径不存在！\n")
+				Output("路径不存在！\n")
 			} else if !stat.IsDir() {
 				return
 			}
 		}
-		Printer("输入有误，请重新输入：")
+		Output("输入有误，请重新输入：")
 	}
 }
 
 func GetText() (text string) {
-	scanner := bufio.NewScanner(os.Stdin)
 	if scanner.Scan() {
 		text = scanner.Text()
 	}
@@ -57,7 +58,6 @@ func GetText() (text string) {
 }
 
 func GetInt() (num int) {
-	scanner := bufio.NewScanner(os.Stdin)
 	for {
 		if scanner.Scan() {
 			text := scanner.Text()
@@ -67,6 +67,6 @@ func GetInt() (num int) {
 				return
 			}
 		}
-		Printer(`请输入正确的数字：`)
+		Output(`请输入正确的数字：`)
 	}
 }

cli/ui/key.go

@@ -19,7 +19,7 @@ func Init() error {
 
 func WaitKeyEnter() {
 	hint := fmt.Sprintf("按下%s继续...\n", cyan(`ENTER`))
-	Printer(hint)
+	Output(hint)
 	for {
 		_, key, _ := keyboard.GetKey()
 		if key == keyboard.KeyEnter {
@@ -41,7 +41,7 @@ func WaitKeyChoose(max int) int {
 
 func WaitKeySwitch() bool {
 	hint := fmt.Sprintf("(%s/%s)\n", cyan(`Y`), cyan(`N`))
-	Printer(hint)
+	Output(hint)
 	for {
 		key, _, _ := keyboard.GetKey()
 		if key == 'y' {

cli/ui/setter.go

@@ -3,11 +3,7 @@ package ui
 import "fmt"
 
 var (
-	Printer = func(text string) {
+	Output = func(text string) {
 		fmt.Print(text)
 	}
 )
-
-func SetPrinter(f func(string)) {
-	Printer = f
-}

go.mod

@@ -7,10 +7,10 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203
 	github.com/gookit/color v1.5.4
+	github.com/k0kubun/pp/v3 v3.2.0
 	github.com/parnurzeal/gorequest v0.3.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tebeka/selenium v0.9.9
-	go.uber.org/zap v1.27.0
 )
 
 require (
@@ -19,13 +19,15 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/moul/http2curl v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -54,6 +54,12 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/k0kubun/pp/v3 v3.2.0 h1:h33hNTZ9nVFNP3u2Fsgz8JXiF5JINoZfFq4SvKJwNcs=
+github.com/k0kubun/pp/v3 v3.2.0/go.mod h1:ODtJQbQcIRfAD3N+theGCV1m/CBxweERz2dapdz1EwA=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
 github.com/parnurzeal/gorequest v0.3.0 h1:SoFyqCDC9COr1xuS6VA8fC8RU7XyrJZN2ona1kEX7FI=
@@ -76,12 +82,6 @@ github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1z
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
-go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
-go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -134,6 +134,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
@@ -149,6 +150,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

net/apps/multithread_downloader/control.go

@@ -0,0 +1,38 @@
+package mdown
+
+import (
+	"net/http"
+	"net/url"
+	"os"
+)
+
+var (
+	TempFilesLocation = "./temp"
+)
+
+const (
+	DefaultMaxThread       = 32
+	DefaultMaxThreadPerURL = 8
+)
+
+type Client struct {
+	Client        *http.Client
+	MaxThread     uint64
+	currentThread uint64
+}
+
+func NewClient(opts ...Option) *Client {
+	client := &Client{
+		Client:        &http.Client{},
+		MaxThread:     DefaultMaxThread,
+		currentThread: 0,
+	}
+	for _, opt := range opts {
+		opt(client)
+	}
+	return client
+}
+
+func (d *Client) Download(url *url.URL, file *os.File, opts ...Option) {
+
+}

net/apps/multithread_downloader/option.go

@@ -0,0 +1,16 @@
+package mdown
+
+import (
+	"net/http"
+	"net/url"
+)
+
+type Option func(*Client)
+
+func WithTransportProxy(proxyURL *url.URL) Option {
+	return func(client *Client) {
+		client.client.Transport = &http.Transport{
+			Proxy: http.ProxyURL(proxyURL),
+		}
+	}
+}

net/apps/multithread_downloader/worker.go

@@ -0,0 +1,31 @@
+package mdown
+
+import (
+	"net/url"
+	"os"
+)
+
+type worker struct {
+	Range reqRange
+	Url   *url.URL
+	File  *os.File
+}
+
+type reqRange struct {
+	Start uint64
+	End   uint64
+}
+
+func newWorker(r reqRange, URL, file string) (*worker, error) {
+	targetURL, err := url.Parse(URL)
+	if err != nil {
+		return nil, err
+	}
+	osFile, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+	return &worker{
+		r, targetURL, osFile,
+	}, nil
+}

net/apps/reverse_proxy/server.go

@@ -7,10 +7,21 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
 	"git.realxlfd.cc/RealXLFD/golib/utils/str"
+	"github.com/gookit/color"
+)
+
+const (
+	LOG_URI_MAX_LENGTH = 35
+)
+
+var (
+	cyan      = color.FgCyan.Render
+	bgMagenta = color.BgBlue.Render
 )
 
 type Server struct {
@@ -20,10 +31,11 @@ type Server struct {
 }
 
 type config struct {
-	Port      int
-	TargetURL *url.URL
-	ProxyURL  *url.URL
-	Handler   func(r *http.Request)
+	Port            int
+	TargetURL       *url.URL
+	ProxyURL        *url.URL
+	RequestHandler  func(r *http.Request)
+	ResponseHandler func(r *http.Response)
 }
 
 func CreateServer(port int, target *url.URL) *Server {
@@ -31,12 +43,28 @@ func CreateServer(port int, target *url.URL) *Server {
 		panic("port out of range (80-65535)")
 	}
 	conf := &config{
-		Port:      port,
-		TargetURL: target,
-		ProxyURL:  nil,
-		Handler:   func(r *http.Request) {},
+		Port:            port,
+		TargetURL:       target,
+		ProxyURL:        nil,
+		RequestHandler:  func(r *http.Request) {},
+		ResponseHandler: func(r *http.Response) {},
 	}
 	reverseProxy := httputil.NewSingleHostReverseProxy(conf.TargetURL)
+	reverseProxy.ModifyResponse = func(r *http.Response) error {
+		Logger.Debug(
+			fmt.Sprintf(
+				"status: %s %s %s",
+				bgMagenta(strconv.Itoa(r.StatusCode)),
+				cyan("<-"),
+				str.TrimMiddle(
+					str.Join(conf.TargetURL.Host, r.Request.RequestURI),
+					LOG_URI_MAX_LENGTH,
+				),
+			),
+		)
+		conf.ResponseHandler(r)
+		return nil
+	}
 	server := &Server{
 		config:         conf,
 		revProxyClient: reverseProxy,
@@ -60,9 +88,15 @@ func (s *Server) WithTransportProxy(proxy string) *Server {
 }
 
 func (s *Server) WithRequestHandler(f func(r *http.Request)) *Server {
-	s.config.Handler = f
+	s.config.RequestHandler = f
 	return s
 }
+
+func (s *Server) WithResponseHandler(f func(r *http.Response)) *Server {
+	s.config.ResponseHandler = f
+	return s
+}
+
 func (s *Server) Run() *Server {
 	if s.httpServer != nil {
 		return s
@@ -111,7 +145,7 @@ func (s *Server) IsRunning() bool {
 
 func (s *Server) Stop() error {
 	if !s.IsRunning() {
-		return errors.New("server has already stopped")
+		return nil
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
@@ -125,22 +159,30 @@ func (s *Server) requestHandler() func(
 	return func(w http.ResponseWriter, r *http.Request) {
 		var proxyInfo string
 		if s.config.ProxyURL != nil {
-			proxyInfo = fmt.Sprintf(" -> [%s]", s.config.ProxyURL.Host)
+			proxyInfo = fmt.Sprintf(
+				" %s [%s]",
+				cyan("->"),
+				s.config.ProxyURL.Host,
+			)
 		}
 		Logger.Info(
 			fmt.Sprintf(
-				"%s -> [:%d]%s -> %s",
+				"%s %s [:%d]%s %s %s",
 				r.RemoteAddr[:strings.Index(r.RemoteAddr, ":")],
+				cyan("->"),
 				s.config.Port,
 				proxyInfo,
+				cyan("->"),
 				str.TrimMiddle(
 					str.Join(s.config.TargetURL.Host, r.RequestURI),
-					35,
+					LOG_URI_MAX_LENGTH,
 				),
 			),
 		)
 		r.Host = s.config.TargetURL.Host
-		s.config.Handler(r)
+		r.URL.Host = s.config.TargetURL.Host
+		r.URL.Scheme = s.config.TargetURL.Scheme
+		s.config.RequestHandler(r)
 		s.revProxyClient.ServeHTTP(w, r)
 	}
 }

net/utils/crawler/parser.go

@@ -3,15 +3,15 @@ package crawler
 import (
 	"errors"
 	"fmt"
-	"git.realxlfd.cc/RealXLFD/golib/net/apis/ym"
-	"git.realxlfd.cc/RealXLFD/golib/net/utils/cookie"
-	"github.com/tebeka/selenium"
 	"math"
 	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
+
+	"git.realxlfd.cc/RealXLFD/golib/net/utils/cookie"
+	"github.com/tebeka/selenium"
 )
 
 type Mode struct {
@@ -241,26 +241,6 @@ func toMode(actions []Action, driver selenium.WebDriver) *Mode {
 					return err
 				},
 			)
-		case VERIFY:
-			executor = append(
-				executor, func() error {
-					script := fmt.Sprintf(scriptPre, action.Dst)
-					data, err := driver.ExecuteScript(script, nil)
-					if err != nil {
-						return err
-					}
-					if imgBase64, ok := data.(string); ok || imgBase64 == "" {
-						// 默认为6位或一下数字字母模式
-						result, errs := ym.CommonVerify(imgBase64, ModeNL8)
-						if errs != nil {
-							return errs[0]
-						}
-						mode.Ctx[action.Save] = result
-						return nil
-					}
-					return errors.New("can not parse image data")
-				},
-			)
 		case WINDOW:
 			executor = append(
 				executor, func() error {
@@ -337,20 +317,9 @@ func parseOperation(line string) (actions Action, err error) {
 		if len(matches) != 2 || matches[1][1] == "" {
 			return Action{}, fmt.Errorf("invalid argument: %s", parts[1])
 		}
-		return Action{Act: WAIT, Dst: matches[0][1], Save: matches[1][1]}, nil
-	}
-	if mark == "VERIFY" {
-		matches := re.FindAllStringSubmatch(line, -1)
-		if len(matches) != 1 || matches[0][1] == "" {
-			return Action{}, fmt.Errorf("invalid argument: %s", parts[1])
-		}
-		replace := fmt.Sprintf(`%s %s `, parts[0], matches[0][0])
-		after := strings.ReplaceAll(line, replace, "")
-		afters := strings.Split(after, " ")
-		if len(afters) != 2 || afters[0] != "->" {
-			return Action{}, fmt.Errorf("invalid argument: %s", after)
-		}
-		return Action{Act: VERIFY, Dst: matches[0][1], Save: afters[1]}, nil
+		return Action{
+			Act: WAIT, Dst: matches[0][1], Save: matches[1][1],
+		}, nil
 	}
 	switch len(parts) {
 	case 1:
@@ -367,7 +336,8 @@ func parseOperation(line string) (actions Action, err error) {
 		case "TO":
 			if strings.HasPrefix(parts[1], ".") {
 				return Action{
-					Act: TO, Dst: "", Save: strings.TrimPrefix(parts[1], "."),
+					Act: TO, Dst: "",
+					Save: strings.TrimPrefix(parts[1], "."),
 				}, nil
 			}
 			return Action{Act: TO, Dst: parts[1]}, nil
@@ -415,7 +385,6 @@ const (
 	WAIT
 	HTML
 	SETCOOKIE
-	VERIFY
 	WINDOW
 	HOVER
 )