使用golang监控目录文件变化

想写个程序,监控目录和文件变化,原先目录非常大,所以感觉要用goroutine对每个目录派生一个goroutine进程,但程序在运行的时候发现,打开的目录非常多,以致系统出错,我们先来看看这个失败的程序,目录小是没有问题的。

// main.go
package main

import (
	//	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	//	"regexp"
	"runtime"
	"strings"

	"github.com/fsnotify/fsnotify"
)

type Fm struct {
	Basedir string
	Watcher *fsnotify.Watcher
	Wdone   chan bool
	Dirchan chan string
}

func (fm *Fm) Init(basedir string) {
	fm.Basedir = filepath.FromSlash(basedir)
	var err error
	fm.Watcher, err = fsnotify.NewWatcher()
	if err != nil {
		log.Fatal("create watcher error:", err)
	}

	fm.Wdone = make(chan bool)

	go func() {
		for {
			select {
			case event := <-fm.Watcher.Events:
				//log.Println("event:", event)
				fm.process_event(event)

			case err := <-fm.Watcher.Errors:
				log.Println("watcher error:", err)
			}
		}
	}()

	fm.Dirchan = make(chan string, 1000)
}

func (fm *Fm) walkdir(path string) {
	//log.Println("basedir:", path)
	fm.Dirchan <- path
	dir, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatal("opendir:", err)
	}
	for _, fi := range dir {

		fpath := filepath.FromSlash(path + "/" + fi.Name())
		if fi.IsDir() {

			if strings.HasPrefix(fi.Name(), ".") {
				continue
			}
			if strings.HasPrefix(fi.Name(), "..") {
				continue
			}
			if strings.Contains(fi.Name(), "lost+found") {
				continue
			}
			go fm.walkdir(fpath)
		} else {
			fm.Dirchan <- fpath
		}
		//log.Println("path:", fpath)
		//ch <- fpath
	}
	//
}
func (fm *Fm) lister() {
	var path string
	for {
		select {
		case path = <-fm.Dirchan:
			err := fm.Watcher.Add(path)
			if err != nil {
				log.Fatal("add watcher error:", err)
			}
		}
	}
}
func (fm *Fm) Start() {
	go fm.walkdir(fm.Basedir)
	go fm.lister()
	defer fm.Watcher.Close()
	<-fm.Wdone
}
func (fm *Fm) process_event(event fsnotify.Event) {
	switch event.Op {
	case fsnotify.Create:
		fm.Watcher.Add(event.Name)
		log.Println("create:", event.Name)
	case fsnotify.Rename, fsnotify.Remove:
		log.Println("remove:", event.Name)
		fm.Watcher.Remove(event.Name)
	case fsnotify.Write:
		log.Println("write:", event.Name)
	}
}
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU() / 2)
	/*
	  echo 50000000 > /proc/sys/fs/inotify/max_user_watches
	  echo 327679 > /proc/sys/fs/inotify/max_queued_events
	*/
	filem := new(Fm)
	filem.Init(os.Args[1])
	filem.Start()
}

上面程序有意思的地方是,递归目录,用多线程进行通讯,但是目录和文件很多的时候,产生的线程也非常多。

下面来个简单的例子,这个例子可以正常工作,速还不错,140G的小文件目录,大约需要1.7G虚拟内存,实占大约500m左右,由于和sersync性能相差太远,所以暂时放弃了这个监控使用。

// main.go
package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/fsnotify/fsnotify"
)

//type MyWatcher *fsnotify.Watcher
func doev(watcher *fsnotify.Watcher, event fsnotify.Event) {
	switch event.Op {
	case fsnotify.Create:
		watcher.Add(event.Name)
		log.Println("create:", event.Name)
	case fsnotify.Rename, fsnotify.Remove:
		log.Println("remove:", event.Name)
		watcher.Remove(event.Name)
	case fsnotify.Write:
		log.Println("write:", event.Name)
	}
}
func main() {
	watchdir := os.Args[1]
	var err error
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}
	defer watcher.Close()

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				//log.Println("event:", event)
				doev(watcher, event)

			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()
	err = watcher.Add(watchdir)
	if err != nil {
		log.Fatal(err)
	}
	err = filepath.Walk(watchdir, func(path string, info os.FileInfo, err error) error {
		err = watcher.Add(path)
		if err != nil {
			log.Fatal(err)
		}
		return nil
	})
	if err != nil {
		fmt.Printf("walk error [%v]\n", err)
	}
	<-done
}

 

用go写windows系统服务

用go+nssm非常容易实现windows系统服务,先看下面的程序

package main

func main() {
 //Call this function where the action happpens
 doStuff()
}

func doStuff() {
 for {
  //the actual stuff happens here.
 }
}

下载nssm复制到c:\windows目录,执行

nssm install MyService d:\MyService.exe

nssm下载地址:http://nssm.cc/description

参考http://sanatgersappa.blogspot.co.id/2013/07/windows-service-with-go-easy-way.html

使用go连接mssql

使用go连接mssql还是比较方便,注意连接字符串要加上

encrypt=disable
package main

import _ "github.com/denisenkom/go-mssqldb"
import "database/sql"
import "log"
import "fmt"
import "flag"

var debug = flag.Bool("debug", false, "enable debugging")
var password = flag.String("password", "", "the database password")
var port *int = flag.Int("port", 1433, "the database port")
var server = flag.String("server", "", "the database server")
var user = flag.String("user", "", "the database user")

func main() {
	flag.Parse() // parse the command line args

	if *debug {
		fmt.Printf(" password:%s\n", *password)
		fmt.Printf(" port:%d\n", *port)
		fmt.Printf(" server:%s\n", *server)
		fmt.Printf(" user:%s\n", *user)
	}

	connString := fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d;encrypt=disable", *server, *user, *password, *port)
	if *debug {
		fmt.Printf(" connString:%s\n", connString)
	}
	conn, err := sql.Open("mssql", connString)
	if err != nil {
		log.Fatal("Open connection failed:", err.Error())
	}
	defer conn.Close()

	stmt, err := conn.Prepare("select 1, 'abc'")
	if err != nil {
		log.Fatal("Prepare failed:", err.Error())
	}
	defer stmt.Close()

	row := stmt.QueryRow()
	var somenumber int64
	var somechars string
	err = row.Scan(&somenumber, &somechars)
	if err != nil {
		log.Fatal("Scan failed:", err.Error())
	}
	fmt.Printf("somenumber:%d\n", somenumber)
	fmt.Printf("somechars:%s\n", somechars)

	fmt.Printf("bye\n")

}

 

使用方法:
E:\Go_Project\mssql_t1>mssql_t1.exe –debug=true –password=123456–server=127.0.0.1 –user=sa

参考

http://studygolang.com/resources/4094

http://www.cnblogs.com/songxingzhu/p/5849148.html

通过http代理安装go包

大家知道,在国内访问国外网站有时候比较困难,安装go的包也很麻烦,所以需要用到http代理了。如果你有一个代理,在windows环境下,执行下面的命令即可使用http代理了。

set https_proxy=127.0.0.1:1080
set http_proxy=127.0.0.1:1080

 

go实现一个简单的http代理

当请求http://localhost:8080/html/home.html 自动转发请求到 http://192.168.0.1/html/home.html,带cookie请求,不过cookie要每次都手工抓取

package main

import (
	"io/ioutil"
	"log"
	"net/http"
	//	"strings"
)

func statistic(w http.ResponseWriter, r *http.Request) {
	//r.URL.RequestURI()
	client := &http.Client{}
	req, err := http.NewRequest("GET", "http://192.168.0.1"+r.URL.Path, nil)
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Set("Cookie", `SessionID=y151QXjszzoP5D2Lvfun0S/JPnFfquDtASLyWz+ZiSg6ngSt1gpakE2DChVN+hnfFOS6rFJ6gwUL+y0pAG+RpYYSogAFcZsuN919FXZle45UGP+ka6fZmceI9ew7Mj4Z;path=/;HttpOnly;`)
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	w.Write(body)
}
func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("/", statistic)
	err := http.ListenAndServe(":8080", mux)

	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

 

go访问sqlite3删除数据

来个增删改查完整版了

删除数据代码如下

package main

import (
	"database/sql"
	"fmt"

	_ "github.com/mattn/go-sqlite3"
)

func main() {
	db, err := sql.Open("sqlite3", "./foo.db")
	checkErr(err)

	stmt, err := db.Prepare("delete from userinfo  where uid=?")
	checkErr(err)
	res, err := stmt.Exec(3)
	checkErr(err)
	affect, err := res.RowsAffected()
	checkErr(err)
	fmt.Println(affect)
}
func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

 

go访问sqlite3更新数据

前面两个文章说了插入,和读取,这个例子演示如何更新数据

package main

import (
	"database/sql"
	"fmt"

	_ "github.com/mattn/go-sqlite3"
)

func main() {
	db, err := sql.Open("sqlite3", "./foo.db")
	checkErr(err)

	stmt, err := db.Prepare("update userinfo set username=? where uid=?")
	checkErr(err)
	res, err := stmt.Exec("kkkkk", 2)
	checkErr(err)
	affect, err := res.RowsAffected()
	checkErr(err)
	fmt.Println(affect)
}
func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

 

go访问sqlite3读取数据

前一文说了如何插入数据,现在可以把数据记录读成一个二维数组,真的太方便了。

package main

import (
	"database/sql"
	"fmt"

	_ "github.com/mattn/go-sqlite3"
)

type userinfo struct {
	id                            int
	username, departname, created string
}

func main() {
	db, err := sql.Open("sqlite3", "./foo.db")
	checkErr(err)
	rows, err := db.Query("select * from userinfo")
	checkErr(err)
	var userlist []userinfo
	var u userinfo
	for rows.Next() {
		rows.Scan(&u.id, &u.username, &u.departname, &u.created)
		userlist = append(userlist, u)
	}
	fmt.Printf("%#v\n", userlist)
	fmt.Printf("len %d\n", len(userlist))
}
func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

输出记录如下

E:\golang\pj2>sqlite4.exe
[]main.userinfo{main.userinfo{id:2, username:"fy", departname:"mydep", created:"2017-01-14T00:00:00Z"}, main.userinfo{id:3, username:"fy", departname:"mydep", created:"2017-01-14T00:00:00Z"}, main.userinfo{id:4, username:"fy", departname:"mydep", created:"2017-01-14T00:00:00Z"}}
len 3

 

go访问sqlite3插入数据

用go访问sqlite真的非常简单,先用sqlite工具创建库,执行sql创建表

   CREATE TABLE `userinfo` (
        `uid` INTEGER PRIMARY KEY AUTOINCREMENT,
        `username` VARCHAR(64) NULL,
        `departname` VARCHAR(64) NULL,
        `created` DATE NULL
    );

    CREATE TABLE `userdeatail` (
        `uid` INT(10) NULL,
        `intro` TEXT NULL,
        `profile` TEXT NULL,
        PRIMARY KEY (`uid`)
    );

go代码如下

package main

import (
	"database/sql"
	"fmt"
	"flag"

	_ "github.com/mattn/go-sqlite3"
)

func main() {
	db, err := sql.Open("sqlite3", "./foo.db")
	checkErr(err)
	sth, err := db.Prepare("INSERT INTO userinfo(username, departname, created) values(?,?,?)")
	checkErr(err)
	res, err := sth.Exec("fy", "www.yiyou.org", "2017-01-14")
	checkErr(err)
	id, _ := res.LastInsertId()

	fmt.Println(id)
	db.Close()
}
func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

 

 

tengine启动脚本

如果你的tengine是独立安装的,想保留原来的nginx,那么这个脚本就可以帮你运行tengine

#! /bin/sh
# chkconfig: 2345 55 25
# Description: Startup script for tengine webserver on Debian. Place in /etc/init.d and
# run 'update-rc.d -f tengine defaults', or use the appropriate command on your
# distro. For CentOS/Redhat run: 'chkconfig --add tengine'

### BEGIN INIT INFO
# Provides:          tengine
# Required-Start:    $all
# Required-Stop:     $all
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: starts the tengine web server
# Description:       starts tengine using start-stop-daemon
### END INIT INFO

# Author:   fy
# website:  http://www.yiyou.org

PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
NAME=nginx
NGINX_BIN=/usr/local/tengine/sbin/$NAME
CONFIGFILE=/usr/local/tengine/conf/$NAME.conf
PIDFILE=/usr/local/tengine/logs/$NAME.pid

case "$1" in
    start)
        echo -n "Starting $NAME... "

        if netstat -tnpl | grep -q nginx;then
            echo "$NAME (pid `pidof $NAME`) already running."
            exit 1
        fi

        $NGINX_BIN -c $CONFIGFILE

        if [ "$?" != 0 ] ; then
            echo " failed"
            exit 1
        else
            echo " done"
        fi
        ;;

    stop)
        echo -n "Stoping $NAME... "

        if ! netstat -tnpl | grep -q nginx; then
            echo "$NAME is not running."
            exit 1
        fi

        $NGINX_BIN -s stop

        if [ "$?" != 0 ] ; then
            echo " failed. Use force-quit"
            exit 1
        else
            echo " done"
        fi
        ;;

    status)
        if netstat -tnpl | grep -q nginx; then
            PID=`pidof nginx`
            echo "$NAME (pid $PID) is running..."
        else
            echo "$NAME is stopped"
            exit 0
        fi
        ;;

    force-quit)
        echo -n "Terminating $NAME... "

        if ! netstat -tnpl | grep -q nginx; then
            echo "$NAME is not running."
            exit 1
        fi

        kill `pidof $NAME`

        if [ "$?" != 0 ] ; then
            echo " failed"
            exit 1
        else
            echo " done"
        fi
        ;;

    restart)
        $0 stop
        sleep 1
        $0 start
        ;;

    reload)
        echo -n "Reload service $NAME... "

        if netstat -tnpl | grep -q nginx; then
            $NGINX_BIN -s reload
            echo " done"
        else
            echo "$NAME is not running, can't reload."
            exit 1
        fi
        ;;

    configtest)
        echo -n "Test $NAME configure files... "

        $NGINX_BIN -t
        ;;

    *)
        echo "Usage: $0 {start|stop|force-quit|restart|reload|status|configtest}"
        exit 1
        ;;

esac