链接一行一个 放link.txt, 怕爬太快被禁ip 没敢搞花活
本来打算js获取(内容不多) go直接爬, 发现页面图片是动态的, 随手记录一下, 没有优化
爬虫
package main
import (
"bufio"
"context"
"fmt"
"io"
"log"
"math"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"unicode"
"github.com/chromedp/chromedp"
"golang.org/x/net/proxy"
)
func readLines(filePath string) ([]string, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer file.Close()
var lines []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
func downloadInsImage(client *http.Client, imgUrl, folder, filename string) error {
resp, err := client.Get(imgUrl)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to download image: %s", imgUrl)
}
out, err := os.Create(filepath.Join(folder, filename))
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
return err
}
func getDatas(ctx context.Context, url string) (string, []string, error) {
var title string
var imageLinks []string
err := chromedp.Run(ctx,
chromedp.Navigate(url),
// 点击加载更多图片按钮直到不存在
chromedp.ActionFunc(func(ctx context.Context) error {
time.Sleep(3 * time.Second)
for {
time.Sleep(500 * time.Millisecond)
var buttonExists bool
err := chromedp.Evaluate(`document.querySelector('div > button._afxw._al46._al47') !== null`, &buttonExists).Do(ctx)
if err != nil {
return err
}
if !buttonExists {
break
}
err = chromedp.Click("div > button._afxw._al46._al47", chromedp.NodeVisible).Do(ctx)
if err != nil {
return err
}
time.Sleep(250 * time.Millisecond)
var imgs []string
if err := chromedp.Run(ctx, chromedp.Evaluate(`Array.from(document.querySelectorAll("._aap0 ._aagv img")).map(img => img.src)`, &imgs)); err != nil {
return nil
}
imageLinks = append(imageLinks, imgs...)
}
return nil
}),
// 获取页面标题
chromedp.Title(&title),
)
if err != nil {
return "", nil, err
}
uniqueLinks := removeDuplicates(imageLinks)
fmt.Printf("共发现: %v张图片\n", len(uniqueLinks))
return title, uniqueLinks, nil
}
func removeDuplicates(imageLinks []string) []string {
encountered := map[string]bool{}
result := []string{}
for v := range imageLinks {
if encountered[imageLinks[v]] == false {
encountered[imageLinks[v]] = true
result = append(result, imageLinks[v])
}
}
return result
}
// 将字符串转换为合法的文件名
func stringToFileName(input string) string {
illegalChars := []rune{'\\', '/', ':', '*', '?', '"', '<', '>', '|'}
illegalCharMap := make(map[rune]bool, len(illegalChars))
for _, char := range illegalChars {
illegalCharMap[char] = true
}
// 过滤掉不允许的字符
var output strings.Builder
for _, char := range input {
if !illegalCharMap[char] && !unicode.IsControl(char) {
output.WriteRune(char)
} else {
output.WriteRune('_') // 用下划线替换非法字符
}
}
out := output.String()
// 文件夹最大名字50字符
minLen := math.Min(float64(len(out)), float64(50))
return output.String()[:int(minLen)]
}
func main() {
output := `E:\Scripting\Program-Learning\test\download`
// 设置代理
proxyURL, err := url.Parse("socks5://127.0.0.1:10808")
if err != nil {
log.Fatalf("failed to parse proxy URL: %v", err)
}
// 创建一个Dialer
dialer, err := proxy.FromURL(proxyURL, proxy.Direct)
if err != nil {
log.Fatalf("failed to create proxy dialer: %v", err)
}
// 创建一个HTTP客户端,使用Dialer
httpTransport := &http.Transport{
Dial: dialer.Dial,
}
client := &http.Client{
Transport: httpTransport,
}
// 读取链接文件
links, err := readLines("links.txt")
if err != nil {
log.Fatalf("Error reading links file: %v", err)
}
if len(links) == 0 {
log.Println("No links found in the file.")
return
}
// 创建Chromedp上下文
ctx, cancel := chromedp.NewExecAllocator(
context.Background(),
append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.Flag("headless", false), // 设置 headless 模式为 false
chromedp.ProxyServer("socks5://127.0.0.1:10808"),
)...,
)
defer cancel()
ctx, cancel = chromedp.NewContext(ctx, chromedp.WithLogf(log.Printf))
defer cancel()
ctx, cancel = context.WithTimeout(ctx, 30*time.Hour)
defer cancel()
index := 1
// 下载每个链接中的图片
for _, link := range links {
folderName, imgsUrl, err := getDatas(ctx, link)
if err != nil {
log.Printf("Error getting data from HTML: %v", err)
continue
}
folderName = stringToFileName(folderName)
fmt.Printf("link: %v\n", link)
targetFolder := filepath.Join(output, folderName)
// 创建文件夹
err = os.MkdirAll(targetFolder, os.ModePerm)
if err != nil {
log.Printf("Error creating folder: %v", err)
continue
}
for _, imgUrl := range imgsUrl {
filename := fmt.Sprintf("image_%d.jpg", index)
err = downloadInsImage(client, imgUrl, targetFolder, filename)
if err != nil {
log.Printf("Error downloading image: %d", index)
} else {
fmt.Printf("Downloaded %d\n", index)
}
index += 1
}
}
}