mirror of
https://github.com/dutchcoders/transfer.sh.git
synced 2024-12-11 05:10:19 +01:00
cb6e5cb0c7
* use dep for vendoring * lets encrypt * moved web to transfer.sh-web repo * single command install * added first tests
65 lines
1.2 KiB
Go
65 lines
1.2 KiB
Go
// Copyright 2013 The Go Authors. All rights reserved.
|
|
//
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file or at
|
|
// https://developers.google.com/open-source/licenses/bsd.
|
|
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/garyburd/redigo/redis"
|
|
"github.com/golang/gddo/database"
|
|
)
|
|
|
|
var crawlCommand = &command{
|
|
name: "crawl",
|
|
run: crawl,
|
|
usage: "crawl [new]",
|
|
}
|
|
|
|
func crawl(c *command) {
|
|
if len(c.flag.Args()) > 1 {
|
|
c.printUsage()
|
|
os.Exit(1)
|
|
}
|
|
db, err := database.New()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
if len(c.flag.Args()) == 1 {
|
|
p, err := ioutil.ReadFile(c.flag.Args()[0])
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
for _, p := range strings.Fields(string(p)) {
|
|
db.AddNewCrawl(p)
|
|
}
|
|
}
|
|
|
|
conn := db.Pool.Get()
|
|
defer conn.Close()
|
|
paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
fmt.Println("NEW")
|
|
for _, path := range paths {
|
|
fmt.Println(path)
|
|
}
|
|
|
|
paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
fmt.Println("BAD")
|
|
for _, path := range paths {
|
|
fmt.Println(path)
|
|
}
|
|
}
|