Db file change

This commit is contained in:
刘河
2019-03-29 15:21:30 +08:00
parent cd7f99063c
commit 5fd335f330
17 changed files with 597 additions and 567 deletions

View File

@@ -1,35 +0,0 @@
package file
import (
"github.com/cnlh/nps/lib/common"
"sort"
"sync"
)
var (
CsvDb *Csv
once sync.Once
)
//init csv from file
func GetCsvDb() *Csv {
once.Do(func() {
CsvDb = NewCsv(common.GetRunPath())
CsvDb.LoadClientFromCsv()
CsvDb.LoadTaskFromCsv()
CsvDb.LoadHostFromCsv()
})
return CsvDb
}
func GetMapKeys(m sync.Map, isSort bool, sortKey, order string) (keys []int) {
if sortKey != "" && isSort {
return sortClientByKey(m, sortKey, order)
}
m.Range(func(key, value interface{}) bool {
keys = append(keys, key.(int))
return true
})
sort.Ints(keys)
return
}

356
lib/file/db.go Normal file
View File

@@ -0,0 +1,356 @@
package file
import (
"errors"
"fmt"
"github.com/cnlh/nps/lib/common"
"github.com/cnlh/nps/lib/crypt"
"github.com/cnlh/nps/lib/rate"
"net/http"
"regexp"
"sort"
"strings"
"sync"
)
type DbUtils struct {
JsonDb *JsonDb
}
var (
Db *DbUtils
once sync.Once
)
//init csv from file
func GetDb() *DbUtils {
once.Do(func() {
jsonDb := NewJsonDb(common.GetRunPath())
jsonDb.LoadClientFromJsonFile()
jsonDb.LoadTaskFromJsonFile()
jsonDb.LoadHostFromJsonFile()
Db = &DbUtils{JsonDb: jsonDb}
})
return Db
}
func GetMapKeys(m sync.Map, isSort bool, sortKey, order string) (keys []int) {
if sortKey != "" && isSort {
return sortClientByKey(m, sortKey, order)
}
m.Range(func(key, value interface{}) bool {
keys = append(keys, key.(int))
return true
})
sort.Ints(keys)
return
}
func (s *DbUtils) GetClientList(start, length int, search, sort, order string, clientId int) ([]*Client, int) {
list := make([]*Client, 0)
var cnt int
keys := GetMapKeys(s.JsonDb.Clients, true, sort, order)
for _, key := range keys {
if value, ok := s.JsonDb.Clients.Load(key); ok {
v := value.(*Client)
if v.NoDisplay {
continue
}
if clientId != 0 && clientId != v.Id {
continue
}
if search != "" && !(v.Id == common.GetIntNoErrByStr(search) || strings.Contains(v.VerifyKey, search) || strings.Contains(v.Remark, search)) {
continue
}
cnt++
if start--; start < 0 {
if length--; length > 0 {
list = append(list, v)
}
}
}
}
return list, cnt
}
func (s *DbUtils) GetIdByVerifyKey(vKey string, addr string) (id int, err error) {
var exist bool
s.JsonDb.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if common.Getverifyval(v.VerifyKey) == vKey && v.Status {
v.Addr = common.GetIpByAddr(addr)
id = v.Id
exist = true
return false
}
return true
})
if exist {
return
}
return 0, errors.New("not found")
}
func (s *DbUtils) NewTask(t *Tunnel) (err error) {
s.JsonDb.Tasks.Range(func(key, value interface{}) bool {
v := value.(*Tunnel)
if (v.Mode == "secret" || v.Mode == "p2p") && v.Password == t.Password {
err = errors.New(fmt.Sprintf("secret mode keys %s must be unique", t.Password))
return false
}
return true
})
if err != nil {
return
}
t.Flow = new(Flow)
s.JsonDb.Tasks.Store(t.Id, t)
s.JsonDb.StoreTasksToJsonFile()
return
}
func (s *DbUtils) UpdateTask(t *Tunnel) error {
s.JsonDb.Tasks.Store(t.Id, t)
s.JsonDb.StoreTasksToJsonFile()
return nil
}
func (s *DbUtils) DelTask(id int) error {
s.JsonDb.Tasks.Delete(id)
s.JsonDb.StoreTasksToJsonFile()
return nil
}
//md5 password
func (s *DbUtils) GetTaskByMd5Password(p string) (t *Tunnel) {
s.JsonDb.Tasks.Range(func(key, value interface{}) bool {
if crypt.Md5(value.(*Tunnel).Password) == p {
t = value.(*Tunnel)
return false
}
return true
})
return
}
func (s *DbUtils) GetTask(id int) (t *Tunnel, err error) {
if v, ok := s.JsonDb.Tasks.Load(id); ok {
t = v.(*Tunnel)
return
}
err = errors.New("not found")
return
}
func (s *DbUtils) DelHost(id int) error {
s.JsonDb.Hosts.Delete(id)
s.JsonDb.StoreHostToJsonFile()
return nil
}
func (s *DbUtils) IsHostExist(h *Host) bool {
var exist bool
s.JsonDb.Hosts.Range(func(key, value interface{}) bool {
v := value.(*Host)
if v.Id != h.Id && v.Host == h.Host && h.Location == v.Location && (v.Scheme == "all" || v.Scheme == h.Scheme) {
exist = true
return false
}
return true
})
return exist
}
func (s *DbUtils) NewHost(t *Host) error {
if t.Location == "" {
t.Location = "/"
}
if s.IsHostExist(t) {
return errors.New("host has exist")
}
t.Flow = new(Flow)
s.JsonDb.Hosts.Store(t.Id, t)
s.JsonDb.StoreHostToJsonFile()
return nil
}
func (s *DbUtils) GetHost(start, length int, id int, search string) ([]*Host, int) {
list := make([]*Host, 0)
var cnt int
keys := GetMapKeys(s.JsonDb.Hosts, false, "", "")
for _, key := range keys {
if value, ok := s.JsonDb.Hosts.Load(key); ok {
v := value.(*Host)
if search != "" && !(v.Id == common.GetIntNoErrByStr(search) || strings.Contains(v.Host, search) || strings.Contains(v.Remark, search)) {
continue
}
if id == 0 || v.Client.Id == id {
cnt++
if start--; start < 0 {
if length--; length > 0 {
list = append(list, v)
}
}
}
}
}
return list, cnt
}
func (s *DbUtils) DelClient(id int) error {
s.JsonDb.Clients.Delete(id)
s.JsonDb.StoreClientsToJsonFile()
return nil
}
func (s *DbUtils) NewClient(c *Client) error {
var isNotSet bool
if c.WebUserName != "" && !s.VerifyUserName(c.WebUserName, c.Id) {
return errors.New("web login username duplicate, please reset")
}
reset:
if c.VerifyKey == "" || isNotSet {
isNotSet = true
c.VerifyKey = crypt.GetRandomString(16)
}
if c.RateLimit == 0 {
c.Rate = rate.NewRate(int64(2 << 23))
c.Rate.Start()
}
if !s.VerifyVkey(c.VerifyKey, c.Id) {
if isNotSet {
goto reset
}
return errors.New("Vkey duplicate, please reset")
}
if c.Id == 0 {
c.Id = int(s.JsonDb.GetClientId())
}
if c.Flow == nil {
c.Flow = new(Flow)
}
s.JsonDb.Clients.Store(c.Id, c)
s.JsonDb.StoreClientsToJsonFile()
return nil
}
func (s *DbUtils) VerifyVkey(vkey string, id int) (res bool) {
res = true
s.JsonDb.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if v.VerifyKey == vkey && v.Id != id {
res = false
return false
}
return true
})
return res
}
func (s *DbUtils) VerifyUserName(username string, id int) (res bool) {
res = true
s.JsonDb.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if v.WebUserName == username && v.Id != id {
res = false
return false
}
return true
})
return res
}
func (s *DbUtils) UpdateClient(t *Client) error {
s.JsonDb.Clients.Store(t.Id, t)
if t.RateLimit == 0 {
t.Rate = rate.NewRate(int64(2 << 23))
t.Rate.Start()
}
return nil
}
func (s *DbUtils) IsPubClient(id int) bool {
client, err := s.GetClient(id)
if err == nil {
return client.NoDisplay
}
return false
}
func (s *DbUtils) GetClient(id int) (c *Client, err error) {
if v, ok := s.JsonDb.Clients.Load(id); ok {
c = v.(*Client)
return
}
err = errors.New("未找到客户端")
return
}
func (s *DbUtils) GetClientIdByVkey(vkey string) (id int, err error) {
var exist bool
s.JsonDb.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if crypt.Md5(v.VerifyKey) == vkey {
exist = true
id = v.Id
return false
}
return true
})
if exist {
return
}
err = errors.New("未找到客户端")
return
}
func (s *DbUtils) GetHostById(id int) (h *Host, err error) {
if v, ok := s.JsonDb.Hosts.Load(id); ok {
h = v.(*Host)
return
}
err = errors.New("The host could not be parsed")
return
}
//get key by host from x
func (s *DbUtils) GetInfoByHost(host string, r *http.Request) (h *Host, err error) {
var hosts []*Host
//Handling Ported Access
host = common.GetIpByAddr(host)
s.JsonDb.Hosts.Range(func(key, value interface{}) bool {
v := value.(*Host)
if v.IsClose {
return true
}
//Remove http(s) http(s)://a.proxy.com
//*.proxy.com *.a.proxy.com Do some pan-parsing
tmp := strings.Replace(v.Host, "*", `\w+?`, -1)
var re *regexp.Regexp
if re, err = regexp.Compile(tmp); err != nil {
return true
}
if len(re.FindAllString(host, -1)) > 0 && (v.Scheme == "all" || v.Scheme == r.URL.Scheme) {
//URL routing
hosts = append(hosts, v)
}
return true
})
for _, v := range hosts {
//If not set, default matches all
if v.Location == "" {
v.Location = "/"
}
if strings.Index(r.RequestURI, v.Location) == 0 {
if h == nil || (len(v.Location) > len(h.Location)) {
h = v
}
}
}
if h != nil {
return
}
err = errors.New("The host could not be parsed")
return
}

View File

@@ -3,21 +3,17 @@ package file
import (
"encoding/json"
"errors"
"fmt"
"github.com/cnlh/nps/lib/common"
"github.com/cnlh/nps/lib/crypt"
"github.com/cnlh/nps/lib/rate"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"sync/atomic"
)
func NewCsv(runPath string) *Csv {
return &Csv{
func NewJsonDb(runPath string) *JsonDb {
return &JsonDb{
RunPath: runPath,
TaskFilePath: filepath.Join(runPath, "conf", "tasks.json"),
HostFilePath: filepath.Join(runPath, "conf", "hosts.json"),
@@ -25,21 +21,21 @@ func NewCsv(runPath string) *Csv {
}
}
type Csv struct {
type JsonDb struct {
Tasks sync.Map
Hosts sync.Map //域名列表
Hosts sync.Map
HostsTmp sync.Map
Clients sync.Map //客户端
RunPath string //存储根目录
ClientIncreaseId int32 //客户端id
TaskIncreaseId int32 //任务自增ID
HostIncreaseId int32 //host increased id
TaskFilePath string
HostFilePath string
ClientFilePath string
Clients sync.Map
RunPath string
ClientIncreaseId int32 //client increased id
TaskIncreaseId int32 //task increased id
HostIncreaseId int32 //host increased id
TaskFilePath string //task file path
HostFilePath string //host file path
ClientFilePath string //client file path
}
func (s *Csv) LoadTaskFromCsv() {
func (s *JsonDb) LoadTaskFromJsonFile() {
loadSyncMapFromFile(s.TaskFilePath, func(v string) {
var err error
post := new(Tunnel)
@@ -56,7 +52,7 @@ func (s *Csv) LoadTaskFromCsv() {
})
}
func (s *Csv) LoadClientFromCsv() {
func (s *JsonDb) LoadClientFromJsonFile() {
loadSyncMapFromFile(s.ClientFilePath, func(v string) {
post := new(Client)
if json.Unmarshal([]byte(v), &post) != nil {
@@ -68,6 +64,7 @@ func (s *Csv) LoadClientFromCsv() {
post.Rate = rate.NewRate(int64(2 << 23))
}
post.Rate.Start()
post.NowConn = 0
s.Clients.Store(post.Id, post)
if post.Id > int(s.ClientIncreaseId) {
s.ClientIncreaseId = int32(post.Id)
@@ -75,7 +72,7 @@ func (s *Csv) LoadClientFromCsv() {
})
}
func (s *Csv) LoadHostFromCsv() {
func (s *JsonDb) LoadHostFromJsonFile() {
loadSyncMapFromFile(s.HostFilePath, func(v string) {
var err error
post := new(Host)
@@ -92,259 +89,7 @@ func (s *Csv) LoadHostFromCsv() {
})
}
func (s *Csv) GetIdByVerifyKey(vKey string, addr string) (id int, err error) {
var exist bool
s.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if common.Getverifyval(v.VerifyKey) == vKey && v.Status {
v.Addr = common.GetIpByAddr(addr)
id = v.Id
exist = true
return false
}
return true
})
if exist {
return
}
return 0, errors.New("not found")
}
func (s *Csv) NewTask(t *Tunnel) (err error) {
s.Tasks.Range(func(key, value interface{}) bool {
v := value.(*Tunnel)
if (v.Mode == "secret" || v.Mode == "p2p") && v.Password == t.Password {
err = errors.New(fmt.Sprintf("Secret mode keys %s must be unique", t.Password))
return false
}
return true
})
if err != nil {
return
}
t.Flow = new(Flow)
s.Tasks.Store(t.Id, t)
s.StoreTasksToCsv()
return
}
func (s *Csv) UpdateTask(t *Tunnel) error {
s.Tasks.Store(t.Id, t)
s.StoreTasksToCsv()
return nil
}
func (s *Csv) DelTask(id int) error {
s.Tasks.Delete(id)
s.StoreTasksToCsv()
return nil
}
//md5 password
func (s *Csv) GetTaskByMd5Password(p string) (t *Tunnel) {
s.Tasks.Range(func(key, value interface{}) bool {
if crypt.Md5(value.(*Tunnel).Password) == p {
t = value.(*Tunnel)
return false
}
return true
})
return
}
func (s *Csv) GetTask(id int) (t *Tunnel, err error) {
if v, ok := s.Tasks.Load(id); ok {
t = v.(*Tunnel)
return
}
err = errors.New("not found")
return
}
func (s *Csv) StoreHostToCsv() {
storeSyncMapToFile(s.Hosts, s.HostFilePath)
}
func (s *Csv) StoreTasksToCsv() {
storeSyncMapToFile(s.Tasks, s.TaskFilePath)
}
func (s *Csv) StoreClientsToCsv() {
storeSyncMapToFile(s.Clients, s.ClientFilePath)
}
func (s *Csv) DelHost(id int) error {
s.Hosts.Delete(id)
s.StoreHostToCsv()
return nil
}
func (s *Csv) GetMapLen(m sync.Map) int {
var c int
m.Range(func(key, value interface{}) bool {
c++
return true
})
return c
}
func (s *Csv) IsHostExist(h *Host) bool {
var exist bool
s.Hosts.Range(func(key, value interface{}) bool {
v := value.(*Host)
if v.Host == h.Host && h.Location == v.Location && (v.Scheme == "all" || v.Scheme == h.Scheme) {
exist = true
return false
}
return true
})
return exist
}
func (s *Csv) NewHost(t *Host) error {
if t.Location == "" {
t.Location = "/"
}
if s.IsHostExist(t) {
return errors.New("host has exist")
}
t.Flow = new(Flow)
s.Hosts.Store(t.Id, t)
s.StoreHostToCsv()
return nil
}
func (s *Csv) GetHost(start, length int, id int, search string) ([]*Host, int) {
list := make([]*Host, 0)
var cnt int
keys := GetMapKeys(s.Hosts, false, "", "")
for _, key := range keys {
if value, ok := s.Hosts.Load(key); ok {
v := value.(*Host)
if search != "" && !(v.Id == common.GetIntNoErrByStr(search) || strings.Contains(v.Host, search) || strings.Contains(v.Remark, search)) {
continue
}
if id == 0 || v.Client.Id == id {
cnt++
if start--; start < 0 {
if length--; length > 0 {
list = append(list, v)
}
}
}
}
}
return list, cnt
}
func (s *Csv) DelClient(id int) error {
s.Clients.Delete(id)
s.StoreClientsToCsv()
return nil
}
func (s *Csv) NewClient(c *Client) error {
var isNotSet bool
if c.WebUserName != "" && !s.VerifyUserName(c.WebUserName, c.Id) {
return errors.New("web login username duplicate, please reset")
}
reset:
if c.VerifyKey == "" || isNotSet {
isNotSet = true
c.VerifyKey = crypt.GetRandomString(16)
}
if c.RateLimit == 0 {
c.Rate = rate.NewRate(int64(2 << 23))
c.Rate.Start()
}
if !s.VerifyVkey(c.VerifyKey, c.Id) {
if isNotSet {
goto reset
}
return errors.New("Vkey duplicate, please reset")
}
if c.Id == 0 {
c.Id = int(s.GetClientId())
}
if c.Flow == nil {
c.Flow = new(Flow)
}
s.Clients.Store(c.Id, c)
s.StoreClientsToCsv()
return nil
}
func (s *Csv) VerifyVkey(vkey string, id int) (res bool) {
res = true
s.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if v.VerifyKey == vkey && v.Id != id {
res = false
return false
}
return true
})
return res
}
func (s *Csv) VerifyUserName(username string, id int) (res bool) {
res = true
s.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if v.WebUserName == username && v.Id != id {
res = false
return false
}
return true
})
return res
}
func (s *Csv) UpdateClient(t *Client) error {
s.Clients.Store(t.Id, t)
if t.RateLimit == 0 {
t.Rate = rate.NewRate(int64(2 << 23))
t.Rate.Start()
}
return nil
}
func (s *Csv) GetClientList(start, length int, search, sort, order string, clientId int) ([]*Client, int) {
list := make([]*Client, 0)
var cnt int
keys := GetMapKeys(s.Clients, true, sort, order)
for _, key := range keys {
if value, ok := s.Clients.Load(key); ok {
v := value.(*Client)
if v.NoDisplay {
continue
}
if clientId != 0 && clientId != v.Id {
continue
}
if search != "" && !(v.Id == common.GetIntNoErrByStr(search) || strings.Contains(v.VerifyKey, search) || strings.Contains(v.Remark, search)) {
continue
}
cnt++
if start--; start < 0 {
if length--; length > 0 {
list = append(list, v)
}
}
}
}
return list, cnt
}
func (s *Csv) IsPubClient(id int) bool {
client, err := s.GetClient(id)
if err == nil {
return client.NoDisplay
}
return false
}
func (s *Csv) GetClient(id int) (c *Client, err error) {
func (s *JsonDb) GetClient(id int) (c *Client, err error) {
if v, ok := s.Clients.Load(id); ok {
c = v.(*Client)
return
@@ -353,84 +98,27 @@ func (s *Csv) GetClient(id int) (c *Client, err error) {
return
}
func (s *Csv) GetClientIdByVkey(vkey string) (id int, err error) {
var exist bool
s.Clients.Range(func(key, value interface{}) bool {
v := value.(*Client)
if crypt.Md5(v.VerifyKey) == vkey {
exist = true
id = v.Id
return false
}
return true
})
if exist {
return
}
err = errors.New("未找到客户端")
return
func (s *JsonDb) StoreHostToJsonFile() {
storeSyncMapToFile(s.Hosts, s.HostFilePath)
}
func (s *Csv) GetHostById(id int) (h *Host, err error) {
if v, ok := s.Hosts.Load(id); ok {
h = v.(*Host)
return
}
err = errors.New("The host could not be parsed")
return
func (s *JsonDb) StoreTasksToJsonFile() {
storeSyncMapToFile(s.Tasks, s.TaskFilePath)
}
//get key by host from x
func (s *Csv) GetInfoByHost(host string, r *http.Request) (h *Host, err error) {
var hosts []*Host
//Handling Ported Access
host = common.GetIpByAddr(host)
s.Hosts.Range(func(key, value interface{}) bool {
v := value.(*Host)
if v.IsClose {
return true
}
//Remove http(s) http(s)://a.proxy.com
//*.proxy.com *.a.proxy.com Do some pan-parsing
tmp := strings.Replace(v.Host, "*", `\w+?`, -1)
var re *regexp.Regexp
if re, err = regexp.Compile(tmp); err != nil {
return true
}
if len(re.FindAllString(host, -1)) > 0 && (v.Scheme == "all" || v.Scheme == r.URL.Scheme) {
//URL routing
hosts = append(hosts, v)
}
return true
})
for _, v := range hosts {
//If not set, default matches all
if v.Location == "" {
v.Location = "/"
}
if strings.Index(r.RequestURI, v.Location) == 0 {
if h == nil || (len(v.Location) > len(h.Location)) {
h = v
}
}
}
if h != nil {
return
}
err = errors.New("The host could not be parsed")
return
func (s *JsonDb) StoreClientsToJsonFile() {
storeSyncMapToFile(s.Clients, s.ClientFilePath)
}
func (s *Csv) GetClientId() int32 {
func (s *JsonDb) GetClientId() int32 {
return atomic.AddInt32(&s.ClientIncreaseId, 1)
}
func (s *Csv) GetTaskId() int32 {
func (s *JsonDb) GetTaskId() int32 {
return atomic.AddInt32(&s.TaskIncreaseId, 1)
}
func (s *Csv) GetHostId() int32 {
func (s *JsonDb) GetHostId() int32 {
return atomic.AddInt32(&s.HostIncreaseId, 1)
}

View File

@@ -23,6 +23,13 @@ func (s *Flow) Add(in, out int64) {
s.ExportFlow += int64(out)
}
type Config struct {
U string
P string
Compress bool
Crypt bool
}
type Client struct {
Cnf *Config
Id int //id
@@ -79,7 +86,7 @@ func (s *Client) GetConn() bool {
}
func (s *Client) HasTunnel(t *Tunnel) (exist bool) {
GetCsvDb().Tasks.Range(func(key, value interface{}) bool {
GetDb().JsonDb.Tasks.Range(func(key, value interface{}) bool {
v := value.(*Tunnel)
if v.Client.Id == s.Id && v.Port == t.Port {
exist = true
@@ -92,7 +99,7 @@ func (s *Client) HasTunnel(t *Tunnel) (exist bool) {
func (s *Client) HasHost(h *Host) bool {
var has bool
GetCsvDb().Hosts.Range(func(key, value interface{}) bool {
GetDb().JsonDb.Hosts.Range(func(key, value interface{}) bool {
v := value.(*Host)
if v.Client.Id == s.Id && v.Host == h.Host && h.Location == v.Location {
has = true
@@ -104,17 +111,17 @@ func (s *Client) HasHost(h *Host) bool {
}
type Tunnel struct {
Id int //Id
Port int //服务端监听端口
Id int
Port int
ServerIp string
Mode string //启动方式
Status bool //设置是否开启
RunStatus bool //当前运行状态
Client *Client //所属客户端id
Ports string //客户端与服务端传递
Mode string
Status bool
RunStatus bool
Client *Client
Ports string
Flow *Flow
Password string //私密模式密码,唯一
Remark string //备注
Password string
Remark string
TargetAddr string
NoStore bool
LocalPath string
@@ -137,10 +144,27 @@ type Health struct {
sync.RWMutex
}
type Host struct {
Id int
Host string //host
HeaderChange string //header change
HostChange string //host change
Location string //url router
Remark string //remark
Scheme string //http https all
NoStore bool
IsClose bool
Flow *Flow
Client *Client
Target *Target //目标
Health `json:"-"`
sync.RWMutex
}
type Target struct {
nowIndex int
TargetStr string
TargetArr []string //目标
TargetArr []string
sync.RWMutex
}
@@ -162,27 +186,3 @@ func (s *Target) GetRandomTarget() (string, error) {
s.nowIndex++
return s.TargetArr[s.nowIndex], nil
}
type Config struct {
U string
P string
Compress bool
Crypt bool
}
type Host struct {
Id int
Host string //host
HeaderChange string //header change
HostChange string //host change
Location string //url router
Remark string //remark
Scheme string //http https all
NoStore bool
IsClose bool
Flow *Flow
Client *Client
Target *Target //目标
Health `json:"-"`
sync.RWMutex
}