From 35e234533c5e85bf48ab04e569d1a518c2e7ff9a Mon Sep 17 00:00:00 2001 From: shoopea Date: Thu, 29 Jun 2023 22:58:24 +0200 Subject: [PATCH] revamp --- addr.go | 112 ++++++ app.go | 1095 ++++++++++++++++++--------------------------------- backup.go | 100 ++--- box.go | 528 ++++++++----------------- config.go | 320 +++++++-------- const.go | 13 + cron.go | 1 + email.go | 82 ++-- go.mod | 6 +- go.sum | 124 ++++++ location.go | 19 - server.go | 44 +++ snapshot.go | 125 +++++- ssh.go | 125 ++++-- utils.go | 41 ++ version.go | 9 +- version.sh | 2 + zfs.go | 341 +++++++++++++++- 18 files changed, 1646 insertions(+), 1441 deletions(-) create mode 100644 addr.go create mode 100644 const.go create mode 100644 cron.go create mode 100644 server.go create mode 100644 utils.go diff --git a/addr.go b/addr.go new file mode 100644 index 0000000..cec14ae --- /dev/null +++ b/addr.go @@ -0,0 +1,112 @@ +package main + +import ( + "errors" + "regexp" + "strings" + + log "github.com/sirupsen/logrus" +) + +type Addr string + +var ( + reBox = regexp.MustCompile(`^[a-zA-Z0-9\-_\.]+$`) + rePath = regexp.MustCompile(`^(/){0,1}[a-zA-Z0-9\-_\.]+(/[a-zA-Z0-9\-_\.]+)+$`) +) + +func (a Addr) Box() string { + s := strings.Split(string(a), `:`) + box := s[0] + if reBox.MatchString(box) { + return box + } else { + return "" + } +} + +func (a Addr) Path() string { + s := strings.Split(string(a), `:`) + path := s[1] + if rePath.MatchString(path) { + return path + } else { + return "" + } +} + +func (a Addr) Append(path string) Addr { + newPath := a.Path() + path + if rePath.MatchString(newPath) { + return Addr(a.Box() + ":" + newPath) + } else { + return "" + } +} + +func (a Addr) BoxExec(cmd string) (string, error) { + log.WithFields(log.Fields{"addr": a}).Debugf("starting") + defer log.WithFields(log.Fields{"addr": a}).Debugf("done") + + if b, ok := cfg.box[a.Box()]; !ok { + err := errors.New("box doesn't exist") + log.WithFields(log.Fields{"addr": a, "box": a.Box()}).Errorf("") + return "", err + } else { + return b.Exec(cmd) + } +} + +func (a Addr) Exec() (string, error) { + log.WithFields(log.Fields{"addr": a}).Debugf("starting") + defer log.WithFields(log.Fields{"addr": a}).Debugf("done") + + return a.BoxExec(a.Path()) +} + +func (a Addr) ValidSnapshots() ([]*ZfsSnapshot, error) { + log.WithFields(log.Fields{"addr": a}).Debugf("starting") + defer log.WithFields(log.Fields{"addr": a}).Debugf("done") + + if b, ok := cfg.box[a.Box()]; !ok { + err := errors.New("box doesn't exist") + log.WithFields(log.Fields{"addr": a, "box": a.Box()}).Errorf("") + return nil, err + } else { + if fs, ok := b.zfs.filesystems[a.Path()]; ok { + return fs.ValidSnapshots(), nil + } else { + err := errors.New("path doesn't exist") + log.WithFields(log.Fields{"addr": a}).Errorf("") + return nil, err + } + } +} + +func (a Addr) Mkdir() error { + log.WithFields(log.Fields{"addr": a}).Debugf("starting") + defer log.WithFields(log.Fields{"addr": a}).Debugf("done") + + if b, ok := cfg.box[a.Box()]; !ok { + err := errors.New("box doesn't exist") + log.WithFields(log.Fields{"addr": a, "box": a.Box()}).Errorf("") + return err + } else { + return b.zfs.Mkdir(a.Path()) + } +} + +func (a Addr) String() string { + return string(a) +} + +func (a Addr) Online() bool { + log.WithFields(log.Fields{"addr": a}).Debugf("starting") + defer log.WithFields(log.Fields{"addr": a}).Debugf("done") + + if b, ok := cfg.box[a.Box()]; !ok { + return false + } else { + return b.online + } +} diff --git a/app.go b/app.go index 7a0d565..1f07568 100644 --- a/app.go +++ b/app.go @@ -1,795 +1,456 @@ package main import ( - "fmt" - "log" + "errors" "regexp" + "strings" "time" + + log "github.com/sirupsen/logrus" ) -type AppConfig struct { - Name string `json:"name"` - Schedule []string `json:"schedule"` - Sources []Location `json:"src"` - Destinations []Location `json:"dest"` - Before map[string]Location `json:"before"` - After map[string]Location `json:"after"` +type App struct { + name string + schedule map[string]struct{} + sources []Addr + destinations []Addr + before map[string]Addr + after map[string]Addr } -func (a AppConfig) getSchedule() (schedule string, err error) { - if *debugFlag { - log.Printf("AppConfig.getSchedule : %s : Start", a.Name) +func (c *Config) NewApp(name string, sources, destinations, schedule []string, before, after map[string]string) (*App, error) { + log.WithFields(log.Fields{"name": name}).Debugf("starting") + defer log.WithFields(log.Fields{"name": name}).Debugf("done") + + a := &App{ + name: name, + sources: make([]Addr, 0), + destinations: make([]Addr, 0), + schedule: make(map[string]struct{}, 0), + before: make(map[string]Addr), + after: make(map[string]Addr), } - refreshSnapshot := make(map[string]bool) - for _, v := range a.Sources { - if !cfg.Box[v.Box()].online { - return "", nil + for _, v := range sources { + src := Addr(v) + if src.Box() == "" { + err := errors.New("source box incorrect") + log.WithFields(log.Fields{"app": name, "addr": v, "error": err}).Errorf("") + return nil, err } - refreshSnapshot[v.Box()] = true - } - for k, _ := range refreshSnapshot { - err := cfg.Box[k].ZFSUpdateSnapshotList() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.getSchedule : %s : ZFSUpdateSnapshotList(%s) : %s", a.Name, k, err) - } - return "", err + + if _, ok := c.box[src.Box()]; !ok { + err := errors.New("source box doesn't exist") + log.WithFields(log.Fields{"app": name, "addr": v, "error": err}).Errorf("") + return nil, err } + + if src.Path() == "" { + err := errors.New("source path incorrect") + log.WithFields(log.Fields{"app": name, "addr": v, "error": err}).Errorf("") + return nil, err + } + + a.sources = append(a.sources, src) } - var ok bool - if *schedFlag != "" { - schedule = *schedFlag - } else if ok, err = a.needYearlySnapshot(); ok && err == nil { - schedule = "yearly" - } else if ok, err = a.needMonthlySnapshot(); ok && err == nil { - schedule = "monthly" - } else if ok, err = a.needWeeklySnapshot(); ok && err == nil { - schedule = "weekly" - } else if ok, err = a.needDailySnapshot(); ok && err == nil { - schedule = "daily" - } else if ok, err = a.needHourlySnapshot(); ok && err == nil { - schedule = "hourly" - } else { - return + for _, v := range destinations { + dest := Addr(v) + if dest.Box() == "" { + err := errors.New("destination box incorrect") + log.WithFields(log.Fields{"app": name, "addr": v, "error": err}).Errorf("") + return nil, err + } + + if _, ok := c.box[dest.Box()]; !ok { + err := errors.New("destination box doesn't exist") + log.WithFields(log.Fields{"app": name, "addr": v, "error": err}).Errorf("") + return nil, err + } + + if dest.Path() == "" { + err := errors.New("destination path incorrect") + log.WithFields(log.Fields{"app": name, "addr": v, "error": err}).Errorf("") + return nil, err + } + + a.destinations = append(a.destinations, dest) } - if ret, ok := cfg.Zfsnap[schedule]; !ok { - schedule = "" - err = fmt.Errorf("no retention for %s", schedule) - } else { - re := regexp.MustCompile(`^([0-9]+[ymwdhMs]{1}|forever)$`) - if !re.MatchString(ret) { - schedule = "" - err = fmt.Errorf("wrong retention format for %s", schedule) + for _, v := range schedule { + switch strings.ToLower(v) { + case "hourly": + a.schedule["hourly"] = struct{}{} + case "daily": + a.schedule["daily"] = struct{}{} + case "weekly": + a.schedule["weekly"] = struct{}{} + case "monthly": + a.schedule["monthly"] = struct{}{} + case "yearly": + a.schedule["yearly"] = struct{}{} + default: + err := errors.New("schedule incorrect") + log.WithFields(log.Fields{"app": name, "schedule": v, "error": err}).Errorf("") + return nil, err } + } - return + + for k, v := range before { + if _, err := regexp.Compile(k); err != nil { + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "call": "regexp.Compile", "attr": k, "error": err}).Errorf("") + return nil, err + } + script := Addr(v) + if script.Box() == "" { + err := errors.New("before box incorrect") + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "error": err}).Errorf("") + return nil, err + } + if script.Path() == "" { + err := errors.New("before path incorrect") + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "error": err}).Errorf("") + return nil, err + } + if _, ok := a.before[k]; ok { + err := errors.New("before already exists") + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "error": err}).Errorf("") + return nil, err + } + + a.before[k] = script + } + + for k, v := range after { + if _, err := regexp.Compile(k); err != nil { + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "call": "regexp.Compile", "attr": k, "error": err}).Errorf("") + return nil, err + } + script := Addr(v) + if script.Box() == "" { + err := errors.New("after box incorrect") + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "error": err}).Errorf("") + return nil, err + } + if script.Path() == "" { + err := errors.New("after path incorrect") + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "error": err}).Errorf("") + return nil, err + } + if _, ok := a.before[k]; ok { + err := errors.New("after already exists") + log.WithFields(log.Fields{"app": name, "schedule": k, "addr": v, "error": err}).Errorf("") + return nil, err + } + + a.after[k] = script + } + + return a, nil } -func (a AppConfig) needYearlySnapshot() (ret bool, err error) { +func (a *App) Cleanup(now time.Time) error { + log.WithFields(log.Fields{"app": a.name}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name}).Debugf("done") - if *debugFlag { - log.Printf("AppConfig.needYearlySnapshot : %s : Start", a.Name) - } - ret = false - - // schedule enabled for app ? - for _, v := range a.Schedule { - if v == "yearly" { - ret = true - } - } - if !ret { - return - } - - // finding out the timestamps existing - timeSource := make(map[string]map[time.Time]struct{}) - timeTotal := make(map[time.Time]struct{}) - re := regexp.MustCompile(`^yearly-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) - for _, src := range a.Sources { - timeSource[string(src)] = make(map[time.Time]struct{}) - - var snapList []Snapshot - snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() - if err != nil { - return - } - - for _, snap := range snapList { - if src.Path() == snap.Path() { - if re.MatchString(snap.Name()) { - dateString := re.ReplaceAllString(snap.Name(), "${Date}") - - var dateTime time.Time - dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.needYearlySnapshot : %s : time.ParseInLocation(%s) : %s", a.Name, dateString, err) - } - return - } else { - timeSource[string(src)][dateTime] = struct{}{} - timeTotal[dateTime] = struct{}{} - } + for _, src := range a.sources { + for _, s := range cfg.box[src.Box()].zfs.filesystems[src.Path()].snapshots { + if expired, err := s.Expired(now); err != nil { + log.WithFields(log.Fields{"app": a.name, "now": now, "box": src.Box(), "snapshot": s.String(), "call": "Expired", "error": err}).Errorf("") + return err + } else if expired { + if err := s.Delete(); err != nil { + log.WithFields(log.Fields{"app": a.name, "now": now, "box": src.Box(), "snapshot": s.String(), "call": "Delete", "error": err}).Errorf("") + return err } } } - } - - // cleaning up the available timestamps for common timestamps - for t, _ := range timeTotal { - for _, v := range timeSource { - if _, ok := v[t]; !ok { - delete(timeTotal, t) - } - } - } - - // finding an eligible timestamp - for t, _ := range timeTotal { - if t.Year() == cfg.Now.Year() { - ret = false - } - } - - // no timestamp => need the snapshot ! - - return -} - -func (a AppConfig) needMonthlySnapshot() (ret bool, err error) { - if *debugFlag { - log.Printf("AppConfig.needMonthlySnapshot : %s : Start", a.Name) - } - ret = false - - // schedule enabled for app ? - for _, v := range a.Schedule { - if v == "monthly" { - ret = true - } - } - if !ret { - return - } - - // finding out the timestamps existing - timeSource := make(map[string]map[time.Time]struct{}) - timeTotal := make(map[time.Time]struct{}) - re := regexp.MustCompile(`^(yearly|monthly)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) - for _, src := range a.Sources { - timeSource[string(src)] = make(map[time.Time]struct{}) - - var snapList []Snapshot - snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() - if err != nil { - return - } - - for _, snap := range snapList { - if src.Path() == snap.Path() { - if re.MatchString(snap.Name()) { - dateString := re.ReplaceAllString(snap.Name(), "${Date}") - - var dateTime time.Time - dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) - } - return - } else { - timeSource[string(src)][dateTime] = struct{}{} - timeTotal[dateTime] = struct{}{} - } - } - } - } - } - - // cleaning up the available timestamps for common timestamps - for t, _ := range timeTotal { - for _, v := range timeSource { - if _, ok := v[t]; !ok { - delete(timeTotal, t) - } - } - } - - // finding an eligible timestamp - for t, _ := range timeTotal { - if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() { - ret = false - } - } - - // no timestamp => need the snapshot ! - - return -} - -func (a AppConfig) needWeeklySnapshot() (ret bool, err error) { - if *debugFlag { - log.Printf("AppConfig.needWeeklySnapshot : %s : Start", a.Name) - } - ret = false - - // schedule enabled for app ? - for _, v := range a.Schedule { - if v == "weekly" { - ret = true - } - } - if !ret { - return - } - - // finding out the timestamps existing - timeSource := make(map[string]map[time.Time]struct{}) - timeTotal := make(map[time.Time]struct{}) - re := regexp.MustCompile(`^(yearly|monthly|weekly)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) - for _, src := range a.Sources { - timeSource[string(src)] = make(map[time.Time]struct{}) - - var snapList []Snapshot - snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() - if err != nil { - return - } - - for _, snap := range snapList { - if src.Path() == snap.Path() { - if re.MatchString(snap.Name()) { - dateString := re.ReplaceAllString(snap.Name(), "${Date}") - - var dateTime time.Time - dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) - } - return - } else { - timeSource[string(src)][dateTime] = struct{}{} - timeTotal[dateTime] = struct{}{} - } - } - } - } - } - - // cleaning up the available timestamps for common timestamps - for t, _ := range timeTotal { - for _, v := range timeSource { - if _, ok := v[t]; !ok { - delete(timeTotal, t) - } - } - } - - // finding an eligible timestamp - nowYear, nowWeek := cfg.Now.ISOWeek() - for t, _ := range timeTotal { - snapYear, snapWeek := t.ISOWeek() - if nowYear == snapYear && nowWeek == snapWeek { - ret = false - } - } - - // no timestamp => need the snapshot ! - - return -} - -func (a AppConfig) needDailySnapshot() (ret bool, err error) { - if *debugFlag { - log.Printf("AppConfig.needDailySnapshot : %s : Start", a.Name) - } - ret = false - - // schedule enabled for app ? - for _, v := range a.Schedule { - if v == "daily" { - ret = true - } - } - if !ret { - return - } - - // finding out the timestamps existing - timeSource := make(map[string]map[time.Time]struct{}) - timeTotal := make(map[time.Time]struct{}) - re := regexp.MustCompile(`^(yearly|monthly|weekly|daily)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) - for _, src := range a.Sources { - timeSource[string(src)] = make(map[time.Time]struct{}) - - var snapList []Snapshot - snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() - if err != nil { - return - } - - for _, snap := range snapList { - if src.Path() == snap.Path() { - if re.MatchString(snap.Name()) { - dateString := re.ReplaceAllString(snap.Name(), "${Date}") - - var dateTime time.Time - dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) - } - return - } else { - timeSource[string(src)][dateTime] = struct{}{} - timeTotal[dateTime] = struct{}{} - } - } - } - } - } - - // cleaning up the available timestamps for common timestamps - for t, _ := range timeTotal { - for _, v := range timeSource { - if _, ok := v[t]; !ok { - delete(timeTotal, t) - } - } - } - - // finding an eligible timestamp - for t, _ := range timeTotal { - if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() { - ret = false - } - } - - // no timestamp => need the snapshot ! - - return -} - -func (a AppConfig) needHourlySnapshot() (ret bool, err error) { - if *debugFlag { - log.Printf("AppConfig.needHourlySnapshot : %s : Start", a.Name) - } - ret = false - - // schedule enabled for app ? - for _, v := range a.Schedule { - if v == "hourly" { - ret = true - } - } - if !ret { - return - } - - // finding out the timestamps existing - timeSource := make(map[string]map[time.Time]struct{}) - timeTotal := make(map[time.Time]struct{}) - re := regexp.MustCompile(`^(yearly|monthly|weekly|daily|hourly)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) - for _, src := range a.Sources { - timeSource[string(src)] = make(map[time.Time]struct{}) - - var snapList []Snapshot - snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() - if err != nil { - return - } - - for _, snap := range snapList { - if src.Path() == snap.Path() { - if re.MatchString(snap.Name()) { - dateString := re.ReplaceAllString(snap.Name(), "${Date}") - - var dateTime time.Time - dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) - } - return - } else { - timeSource[string(src)][dateTime] = struct{}{} - timeTotal[dateTime] = struct{}{} - } - } - } - } - } - - // cleaning up the available timestamps for common timestamps - for t, _ := range timeTotal { - for _, v := range timeSource { - if _, ok := v[t]; !ok { - delete(timeTotal, t) - } - } - } - - // finding an eligible timestamp - for t, _ := range timeTotal { - if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() && t.Hour() == cfg.Now.Hour() { - ret = false - } - } - - // no timestamp => need the snapshot ! - - return -} - -func (a AppConfig) CheckZFS() error { - if *debugFlag { - log.Printf("AppConfig.CheckZFS : %s : Start", a.Name) - } - - for _, src := range a.Sources { - if !cfg.Box[src.Box()].ZFSIsZFS(src.Path()) { - return fmt.Errorf("No path %s on source", string(src)) - } - for _, dest := range a.Destinations { - if cfg.Box[dest.Box()].online { - if !cfg.Box[dest.Box()].ZFSIsZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()) { - err := cfg.Box[dest.Box()].ZFSCreateZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.CheckZFS : %s : Error creating %s on %s", a.Name, dest.Path()+"/"+src.Box()+"/"+src.Path(), dest.Box()) - } + for _, dest := range a.destinations { + dest = dest.Append("/" + src.Box() + "/" + src.Path()) + for _, s := range cfg.box[dest.Box()].zfs.filesystems[dest.Path()].snapshots { + if expired, err := s.Expired(now); err != nil { + log.WithFields(log.Fields{"app": a.name, "now": now, "box": dest.Box(), "snapshot": s.String(), "call": "Expired", "error": err}).Errorf("") + return err + } else if expired { + if err := s.Delete(); err != nil { + log.WithFields(log.Fields{"app": a.name, "now": now, "box": src.Box(), "snapshot": s.String(), "call": "Delete", "error": err}).Errorf("") return err } } } } } + return nil } -func (a AppConfig) ExecBefore(schedule string) error { - if *debugFlag { - log.Printf("AppConfig.ExecBefore : %s : Start %s", a.Name, schedule) - } +func (a *App) SanityCheck() error { + log.WithFields(log.Fields{"app": a.name}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name}).Debugf("done") - for k, v := range a.Before { - re := regexp.MustCompile(k) - if re.MatchString(schedule) { - _, err := cfg.Box[v.Box()].SSHExec(v.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.ExecBefore : %s : Error executing %s", a.Name, string(v)) - } - return err - } + for _, src := range a.sources { + b := cfg.box[src.Box()] + if !b.online { + err := errors.New("source box offline") + log.WithFields(log.Fields{"app": a.name, "box": src.Box(), "error": err}).Errorf("") + return err } - } - return nil -} -func (a AppConfig) ExecAfter(schedule string) error { - if *debugFlag { - log.Printf("AppConfig.ExecAfter : %s : Start %s", a.Name, schedule) - } - - for k, v := range a.After { - re := regexp.MustCompile(k) - if re.MatchString(schedule) { - _, err := cfg.Box[v.Box()].SSHExec(v.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.ExecAfter : %s : Error executing %s on %s", a.Name, v.Path(), v.Box()) - } - return err - } - } - } - - return nil -} - -func (a AppConfig) TakeSnapshot(schedule string) error { - if *debugFlag { - log.Printf("AppConfig.TakeSnapshot : %s : Start %s", a.Name, schedule) - } - - for _, v := range a.Sources { - err := cfg.Box[v.Box()].ZFSTakeSnapshot(schedule, v.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.TakeSnapshot : %s : ZFSTakeSnapshot", a.Name) - } + if _, ok := b.zfs.filesystems[src.Path()]; !ok { + err := errors.New("source path doesn't exist") + log.WithFields(log.Fields{"app": a.name, "box": src.Box(), "path": src.Path(), "error": err}).Errorf("") return err } } + onlineDestinations := 0 + for _, dest := range a.destinations { + if cfg.box[dest.Box()].online { + onlineDestinations++ + } + } + if onlineDestinations == 0 { + err := errors.New("no destination box online") + log.WithFields(log.Fields{"app": a.name, "error": err}).Errorf("") + return err + } + return nil } -func (a AppConfig) RefreshSnapshot() error { - if *debugFlag { - log.Printf("AppConfig.RefreshSnapshot : %s : Start", a.Name) +func (a *App) RunSchedule(schedule string, now time.Time) error { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "now": now}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "now": now}).Debugf("done") + + snapshotName := SnapshotName(schedule, now) + log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "now": now, "snapshot": snapshotName}).Debugf("snapshot name") + + if err := a.RunBefore(schedule); err != nil { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "now": now, "call": "RunBefore", "attr": schedule, "error": err}).Errorf("") } - refreshSnapshot := make(map[string]struct{}) - for _, v := range a.Sources { - if cfg.Box[v.Box()].online { - refreshSnapshot[v.Box()] = struct{}{} + for _, src := range a.sources { + srcFs := cfg.box[src.Box()].zfs.filesystems[src.Path()] + if _, err := srcFs.TakeSnapshot(snapshotName); err != nil { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "now": now, "call": "TakeSnapshot", "attr": snapshotName, "error": err}).Errorf("") } + } - for _, v := range a.Destinations { - if cfg.Box[v.Box()].online { - refreshSnapshot[v.Box()] = struct{}{} - } + + if err := a.RunAfter(schedule); err != nil { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "now": now, "call": "RunAfter", "attr": schedule, "error": err}).Errorf("") } - for k, _ := range refreshSnapshot { - if *debugFlag { - log.Printf("AppConfig.RefreshSnapshot : %s : refreshing snapshots for source %s", a.Name, k) - } - err := cfg.Box[k].ZFSUpdateSnapshotList() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RefreshSnapshot : %s : Error getting snapshots on %s", a.Name, k) - } + + return nil + +} + +func (a *App) Run(now time.Time) error { + log.WithFields(log.Fields{"app": a.name, "now": now}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name, "now": now}).Debugf("done") + + if err := a.SanityCheck(); err != nil { + log.WithFields(log.Fields{"app": a.name, "now": now, "call": "SanityCheck", "error": err}).Errorf("") + return err + } + + schedule, err := a.NextSchedule(now) + if err != nil { + log.WithFields(log.Fields{"app": a.name, "call": "NextSchedule", "error": err}).Errorf("") + return err + } + + log.WithFields(log.Fields{"app": a.name, "now": now, "schedule": schedule}).Debugf("schedule") + if schedule != "" { + if err := a.RunSchedule(schedule, now); err != nil { + log.WithFields(log.Fields{"app": a.name, "call": "NextSchedule", "error": err}).Errorf("") return err } } + + if err := a.Transfer(); err != nil { + log.WithFields(log.Fields{"app": a.name, "call": "Transfer", "error": err}).Errorf("") + return err + } + + if err := a.Cleanup(now); err != nil { + log.WithFields(log.Fields{"app": a.name, "call": "Cleanup", "error": err}).Errorf("") + return err + } + return nil } -func (a AppConfig) CreatePath() (err error) { - if *debugFlag { - log.Printf("AppConfig.CreatePath : %s : Start", a.Name) - } +func (a *App) NextSchedule(now time.Time) (string, error) { + log.WithFields(log.Fields{"app": a.name, "now": now}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name, "now": now}).Debugf("done") - for _, src := range a.Sources { - for _, dest := range a.Destinations { - if cfg.Box[dest.Box()].online { - if *debugFlag { - log.Printf("AppConfig.CreatePath : %s : Checking path on %s", a.Name, string(dest)) - } + // get a list of all the common timestamps in sources snapshots + snapshots := make(map[string]map[time.Time]int) + for _, v := range a.sources { + log.WithFields(log.Fields{"app": a.name, "now": now, "box": v.Box(), "path": v.Path()}).Debugf("source") + b := cfg.box[v.Box()] // we tested the boxes - if !cfg.Box[dest.Box()].ZFSIsZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()) { - if *debugFlag { - log.Printf("AppConfig.CreatePath : %s : Creating on %s path %s", a.Name, dest.Box(), dest.Path()+"/"+src.Box()+"/"+src.Path()) - } - if err = cfg.Box[dest.Box()].ZFSCreateZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()); err != nil { - if *debugFlag { - log.Printf("AppConfig.CreatePath : %s : Creating on %s path %s failed (%s)", a.Name, dest.Box(), dest.Path()+"/"+src.Box()+"/"+src.Path(), err) - } - return - } - } - } + fs, ok := b.zfs.filesystems[v.Path()] + if !ok { + err := errors.New("path doesn't exist") + log.WithFields(log.Fields{"app": a.name, "now": now, "box": v.Box(), "path": v.Path(), "error": err}).Errorf("") + return "", err } - } - err = nil - return -} -func (a AppConfig) SendSnapshots() (err error) { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Start", a.Name) - } + log.WithFields(log.Fields{"app": a.name, "now": now, "box": v.Box(), "path": v.Path()}).Debugf("%d snapshots", len(fs.snapshots)) - for _, src := range a.Sources { - if cfg.Box[src.Box()].online { - for _, dest := range a.Destinations { - if cfg.Box[dest.Box()].online { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Sending snapshots from %s to %s", a.Name, string(src), string(dest)) + for _, v2 := range fs.snapshots { + if s, err := v2.Schedule(); err == nil { + snapshots2, ok := snapshots[s] + if !ok { + snapshots2 = make(map[time.Time]int) + } + + if t, err := v2.Timestamp(); err == nil { + if t.After(now) { + err := errors.New("snapshot in the future") + log.WithFields(log.Fields{"app": a.name, "now": now, "source": v, "timestamp": t, "error": err}).Errorf("") + return "", err } - - var dLastSnapshot Snapshot - dLastSnapshot, err = cfg.Box[dest.Box()].ZFSGetLastSnapshot(dest.Path() + "/" + src.Box() + "/" + src.Path()) - if err != nil && err.Error() == "no snapshot" { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s on %s", a.Name, string(src), dest.Box()) - } - - var sFirstSnapshot Snapshot - sFirstSnapshot, err = cfg.Box[src.Box()].ZFSGetFirstSnapshot(src.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s", a.Name, string(src)) - } - return - } - - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s", a.Name, dest.Box(), string(sFirstSnapshot)) - } - _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + cfg.Box[src.Box()].Host() + " zfs send " + string(sFirstSnapshot) + " | zfs recv -F " + dest.Path() + "/" + src.Box() + "/" + src.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sFirstSnapshot), err) - } - return - } - - var ( - sCurrSnapshot, sNextSnapshot Snapshot - isLastSnapshot bool - ) - sNextSnapshot = sFirstSnapshot - isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) - if !isLastSnapshot { - sCurrSnapshot = sNextSnapshot - sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetLastSnapshot(sNextSnapshot.Path()) - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box()) - } - if err != nil && err.Error() != "no snapshot" { - return - } - _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + cfg.Box[src.Box()].Host() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv -F " + dest.Path() + "/" + src.Box() + "/" + src.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err) - } - return - } - } - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : All snapshots sent for %s", a.Name, string(src)) - } - + if count, ok := snapshots2[t]; ok { + snapshots2[t] = count + 1 } else { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Last snapshot on %s is %s", a.Name, dest.Box(), string(dLastSnapshot)) - } - var ( - sCurrSnapshot, sNextSnapshot Snapshot - isLastSnapshot bool - ) - sNextSnapshot = Snapshot(string(dLastSnapshot)[len(dest.Path())+len(src.Box())+2:]) - isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) - if !isLastSnapshot { - sCurrSnapshot = sNextSnapshot - sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetLastSnapshot(sNextSnapshot.Path()) - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box()) - } - if err != nil && err.Error() != "no snapshot" { - return - } - _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + cfg.Box[src.Box()].Host() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv -F " + dest.Path() + "/" + src.Box() + "/" + src.Path()) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err) - } - return - } - } + snapshots2[t] = 1 } } + + snapshots[s] = snapshots2 } } } - err = nil - return -} -func (a AppConfig) CleanupSnapshot() error { - if *debugFlag { - log.Printf("AppConfig.CleanupSnapshot : %s : Start", a.Name) + t := time.Unix(0, 0) + if _, ok := a.schedule["yearly"]; ok { + if s, ok := snapshots["yearly"]; ok { + for k, v := range s { + if k.After(t) && v == len(a.sources) { + t = k + } + } + } + if t.Year() < now.Year() { + return "yearly", nil + } } - cleanupSnapshot := make(map[string]string) + if _, ok := a.schedule["monthly"]; ok { + if s, ok := snapshots["monthly"]; ok { + for k, v := range s { + if k.After(t) && v == len(a.sources) { + t = k + } + } + } + if t.Year() < now.Year() || t.Month() != now.Month() { + return "month", nil + } + } - for _, dest := range a.Destinations { - if cfg.Box[dest.Box()].online { - for _, src := range a.Sources { - if cfg.Box[src.Box()].online { - cleanupSnapshot[src.Box()] = cleanupSnapshot[src.Box()] + " " + src.Path() - cleanupSnapshot[dest.Box()] = cleanupSnapshot[dest.Box()] + " " + dest.Path() + "/" + src.Box() + "/" + src.Path() + if _, ok := a.schedule["weekly"]; ok { + if s, ok := snapshots["weekly"]; ok { + for k, v := range s { + if k.After(t) && v == len(a.sources) { + t = k + } + } + } + ny, nw := now.ISOWeek() + ty, tw := t.ISOWeek() + if ty < ny || tw < nw { + return "month", nil + } + } + + if _, ok := a.schedule["daily"]; ok { + if s, ok := snapshots["daily"]; ok { + for k, v := range s { + if k.After(t) && v == len(a.sources) { + t = k + } + } + } + if t.Year() < now.Year() || t.Month() != now.Month() || t.Day() < now.Day() { + return "daily", nil + } + } + + if _, ok := a.schedule["hourly"]; ok { + if s, ok := snapshots["hourly"]; ok { + for k, v := range s { + if k.After(t) && v == len(a.sources) { + t = k + } + } + } + if t.Year() < now.Year() || t.Month() != now.Month() || t.Day() < now.Day() || t.Hour() < now.Hour() { + return "hourly", nil + } + } + + return "", nil +} + +func (a *App) RunBefore(schedule string) error { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name, "schedule": schedule}).Debugf("done") + + for k, v := range a.before { + re := regexp.MustCompile(k) + if re.MatchString(schedule) { + if _, err := v.Exec(); err != nil { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "regex": k, "call": "Exec", "error": err}).Errorf("") + return err + } + } + } + + return nil +} + +func (a *App) RunAfter(schedule string) error { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name, "schedule": schedule}).Debugf("done") + + for k, v := range a.after { + re := regexp.MustCompile(k) + if re.MatchString(schedule) { + if _, err := v.Exec(); err != nil { + log.WithFields(log.Fields{"app": a.name, "schedule": schedule, "regex": k, "call": "Exec", "error": err}).Errorf("") + return err + } + } + } + + return nil +} + +func (a *App) Transfer() error { + log.WithFields(log.Fields{"app": a.name}).Debugf("starting") + defer log.WithFields(log.Fields{"app": a.name}).Debugf("done") + + for _, src := range a.sources { + for _, dest := range a.destinations { + dest := dest.Append("/" + src.Box() + "/" + src.Path()) + if dest.Online() { + if err := dest.Mkdir(); err != nil { + log.WithFields(log.Fields{"app": a.name, "call": "Mkdir", "attr": dest, "error": err}).Errorf("") + return err + } + + if err := TransferZfs(src, dest); err != nil { + log.WithFields(log.Fields{"app": a.name, "call": "TransferZfs", "src": src, "dest": dest, "error": err}).Errorf("") + return err } } } } - for k, v := range cleanupSnapshot { - if *debugFlag { - log.Printf("AppConfig.CleanupSnapshot : %s : cleaning snapshots on %s for %s", a.Name, k, v) - } - _, err := cfg.Box[k].SSHExec("zfsnap destroy -p hourly- -p daily- -p weekly- -p monthly- -p yearly-" + v) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.CleanupSnapshot : %s : Error executing zfsnap on %s", a.Name, k) - } - return err - } - } - - return nil -} - -func (a AppConfig) RunAppBackup() error { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : Start", a.Name) - } - schedule, err := a.getSchedule() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : Error getting schedule : %s", a.Name, err) - } - return err - } - - if schedule != "" || *slowFlag { - err = a.CheckZFS() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : CheckZFS : %s", a.Name, err) - } - return err - } - } - - err = a.CleanupSnapshot() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : CleanupSnapshot : %s", a.Name, err) - } - return err - } - - if schedule != "" { - err = a.ExecBefore(schedule) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : ExecBefore : %s", a.Name, err) - } - return err - } - - err = a.TakeSnapshot(schedule) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : TakeSnapshot : %s", a.Name, err) - } - return err - } - } - - err = a.RefreshSnapshot() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : RefreshSnapshot : %s", a.Name, err) - } - return err - } - - err = a.CreatePath() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : CreatePath : %s", a.Name, err) - } - return err - } - - err = a.SendSnapshots() - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : SendSnapshots : %s", a.Name, err) - } - return err - } - - if schedule != "" { - err = a.ExecAfter(schedule) - if err != nil { - if *debugFlag { - log.Printf("AppConfig.RunAppBackup : %s : ExecAfter : %s", a.Name, err) - } - return err - } - } - return nil } diff --git a/backup.go b/backup.go index a2214e9..53316b4 100644 --- a/backup.go +++ b/backup.go @@ -3,22 +3,19 @@ package main import ( "flag" "fmt" - "log" "os" "time" + + log "github.com/sirupsen/logrus" ) var ( - appFlag = flag.String("app", "", "run specific app") - cfgFile = flag.String("config", "config.json", "config file") - schedFlag = flag.String("schedule", "", "specific schedule") - slowFlag = flag.Bool("slow", false, "slow process") - testFlag = flag.Bool("test", false, "test run") - debugFlag = flag.Bool("debug", false, "debug") - testMailFlag = flag.Bool("test-mail", false, "test email setup") - stopOnErrorFlag = flag.Bool("stop-on-error", false, "stop processing on error") - cfg Config - email *Email + cfgFile = flag.String("config", "config.json", "config file") + isDaemon = flag.Bool("daemon", false, "run as daemon") + debug = flag.Bool("debug", false, "log debug messages") + logFile = flag.String("logfile", "", "log file") + cfg Config + email *Email ) func main() { @@ -26,70 +23,41 @@ func main() { fmt.Printf("backup (%s)\n", version) + if *debug { + log.SetLevel(log.DebugLevel) + } + if *logFile != "" { + if f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644); err != nil { + log.Printf("Cannot open logfile (%s)", err) + } else { + log.SetOutput(f) + defer f.Close() + } + + } + log.SetReportCaller(true) + email = new(Email) email.startTime = time.Now() email.items = make([]string, 0) - err := cfg.Load() - if err != nil { + if err := cfg.LoadFile(*cfgFile); err != nil { log.Printf("Cannot load config (%s)", err) os.Exit(1) } - if *testMailFlag { - SendMail(cfg.Email.SmtpHost, cfg.Email.FromEmail, "test backup email topic", "test backup email body", cfg.Email.ToEmail) + if *isDaemon { + cfg.Start(nil) + server := NewServer(cfg.Admin.Addr, cfg.Admin.Username, cfg.Admin.Password) + server.Run() + } else { + e := NewEmail(time.Now()) + cfg.Start(e) + defer cfg.Stop(e) + + cfg.Run(e) + os.Exit(0) } - err = RunBackup(*appFlag, *stopOnErrorFlag) - if err != nil { - log.Printf("Cannot run schedule (%s)", err) - os.Exit(1) - } - - if len(email.items) > 0 { - body := " - " + email.items[0] - for _, v := range email.items[1:] { - body = body + "\r\n" + " - " + v - } - SendMail(cfg.Email.SmtpHost, cfg.Email.FromEmail, "Autobackup report", body, cfg.Email.ToEmail) - log.Printf("Sending summary email\r\n%v", email.items) - } -} - -//RunBackup run all backup targets where schedule is registered -func RunBackup(app string, stopOnError bool) error { - if app == "" { - if *debugFlag { - log.Printf("RunBackup() : Start") - } - for _, a := range cfg.Apps { - err := a.RunAppBackup() - if err != nil { - if *debugFlag { - log.Printf("RunBackup() : Error running %s", a.Name) - } - if stopOnError { - return err - } - } - } - } else { - if *debugFlag { - log.Printf("RunBackup() : Start %s", app) - } - for _, a := range cfg.Apps { - if a.Name == app { - err := a.RunAppBackup() - if err != nil { - if *debugFlag { - log.Printf("RunBackup() : Error running %s", a.Name) - } - return err - } - } - } - } - - return nil } diff --git a/box.go b/box.go index c7545bb..7948c67 100644 --- a/box.go +++ b/box.go @@ -1,423 +1,205 @@ package main import ( - "bytes" - "encoding/csv" - "fmt" - "log" - "strings" + "errors" + "regexp" + "sync" + + "github.com/silenceper/pool" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" ) type Box struct { - Addr string `json:"addr"` - User string `json:"user"` - Key string `json:"key"` - Name string `json:"-"` - ssh *SSHConfig - zfs *ZFSConfig - online bool + name string + addr string + user string + key string + zfs *BoxZfs + sshPool pool.Pool + created bool + online bool + mx sync.Mutex } -func (b *Box) ZFSTakeSnapshot(schedule, path string) (err error) { - if *debugFlag { - log.Printf("Box.ZFSTakeSnapshot : %s : Taking snapshot on %s for %s", b.Name, path, schedule) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - err = b.SnapshotInitialize() - if err != nil { - return - } - - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - - timestamp := cfg.Now.Format("2006-01-02_15.04.05") - name := fmt.Sprintf("%s-%s--%s", schedule, timestamp, cfg.Zfsnap[schedule]) - _, err = b.ssh.exec("zfs snapshot " + path + "@" + name) - - if err != nil { - return - } - - b.zfs.SnapshotAdded = true - b.zfs.SnapshotList = append(b.zfs.SnapshotList, Snapshot(path+"@"+name)) - - return +type BoxSshPool struct { + signer ssh.Signer + config *ssh.ClientConfig + client *ssh.Client + logged bool + mx sync.Mutex } -func (b *Box) ZFSGetLastSnapshot(path string) (last Snapshot, err error) { - if *debugFlag { - log.Printf("Box.ZFSGetLastSnapshot : %s : Start %s (%d snapshots)", b.Name, path, len(b.zfs.SnapshotList)) +func (c *Config) NewBox(name, addr, user, key string) (b *Box, err error) { + log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key}).Debugf("starting") + defer log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key}).Debugf("done") + + re := regexp.MustCompile(boxNamePattern) + if !re.MatchString(name) { + err := errors.New("invalid name") + log.WithFields(log.Fields{"name": b.name, "error": err}).Errorf("") + return nil, err } - if !b.online { - err = fmt.Errorf("box offline") - return - } - - err = b.SnapshotInitialize() + p, err := NewSshPool(name, addr, user, key) if err != nil { - return last, err + log.WithFields(log.Fields{"name": b.name, "call": "NewSshPool", "error": err}).Errorf("") + return nil, err } - b.zfs.M.Lock() - defer b.zfs.M.Unlock() + b = &Box{ + name: name, + addr: addr, + user: user, + key: key, + zfs: &BoxZfs{ + online: false, + }, + sshPool: p, + online: false, + created: true, + } - for _, v := range b.zfs.SnapshotList { - if v.Path() == path { - last = v - } - } - if len(string(last)) == 0 { - err = fmt.Errorf("no snapshot") - } - return + b.zfs.box = b + + return b, nil } -func (b *Box) ZFSIsLastSnapshot(src Snapshot) (is bool, err error) { - if *debugFlag { - log.Printf("Box.ZFSIsLastSnapshot : %s : Start %s", b.Name, string(src)) - } +func (b *Box) Open() error { + log.WithFields(log.Fields{"name": b.name}).Debugf("starting") + defer log.WithFields(log.Fields{"name": b.name}).Debugf("done") - if !b.online { - err = fmt.Errorf("box offline") - return - } + b.mx.Lock() + defer b.mx.Unlock() - err = b.SnapshotInitialize() - if err != nil { - return - } - - _, err = b.ZFSGetNextSnapshot(src) - if err != nil { - if err.Error() == "no snapshot" { - is = true - err = nil - } - } else { - is = false - } - return -} - -func (b *Box) ZFSGetFirstSnapshot(path string) (first Snapshot, err error) { - if *debugFlag { - log.Printf("Box.ZFSGetFirstSnapshot : %s : Start %s", b.Name, path) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - err = b.SnapshotInitialize() - if err != nil { - return - } - - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - - for _, v := range b.zfs.SnapshotList { - if v.Path() == path { - first = v - return - } - } - err = fmt.Errorf("no snapshot") - return -} - -func (b *Box) ZFSGetNextSnapshot(src Snapshot) (next Snapshot, err error) { - if *debugFlag { - log.Printf("Box.ZFSGetNextSnapshot : %s : Start %s", b.Name, string(src)) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - err = b.SnapshotInitialize() - if err != nil { - return - } - - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - - for id, v := range b.zfs.SnapshotList { - if v == src { - if len(b.zfs.SnapshotList) > id+1 { - next = b.zfs.SnapshotList[id+1] - if next.Path() == src.Path() { - return - } else { - err = fmt.Errorf("no snapshot") - return - } - } else { - err = fmt.Errorf("no snapshot") - return - } - } - } - err = fmt.Errorf("no snapshot") - return -} - -func (b *Box) ZFSUpdateSnapshotList() (err error) { - if *debugFlag { - log.Printf("Box.ZFSUpdateSnapshotList : %s : Start", b.Name) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - b.zfs.M.Lock() - if b.zfs.SnapshotDeleted || b.zfs.SnapshotAdded { - b.zfs.SnapshotInitialized = false - } - b.zfs.M.Unlock() - - err = b.SnapshotInitialize() - return -} - -func (b *Box) ZFSGetSnapshotList() (snaps []Snapshot, err error) { - if *debugFlag { - log.Printf("Box.ZFSGetSnapshotList : %s : Start", b.Name) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - err = b.SnapshotInitialize() - if err != nil { - return - } - - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - - snaps = b.zfs.SnapshotList - return -} - -func (b *Box) SnapshotInitialize() (err error) { - if *debugFlag { - log.Printf("Box.SnapshotInitialize : %s : Start", b.Name) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - - if b.zfs.SnapshotInitialized { + if b.online { return nil } - if *debugFlag { - log.Printf("Box.SnapshotInitialize : %s : Start", b.Name) - } - - b.zfs.SnapshotList = make([]Snapshot, 0) - - var buf *bytes.Buffer - buf, err = b.SSHExec("zfs list -H -t snapshot -o name") - - csvReader := csv.NewReader(buf) - csvReader.Comma = '\t' - csvReader.FieldsPerRecord = 1 - - csvData, err := csvReader.ReadAll() + hostname, err := b.Exec("hostname") if err != nil { - if *debugFlag { - log.Printf("Box.SnapshotInitialize : %s : csvReader.ReadAll() : %s", b.Name, err) - } + log.WithFields(log.Fields{"name": b.name, "call": "Exec", "attr": "hostname", "error": err}).Errorf("") return err } - for _, rec := range csvData { - b.zfs.SnapshotList = append(b.zfs.SnapshotList, Snapshot(rec[0])) - } + log.WithFields(log.Fields{"name": b.name}).Debugf("hostname : %s", hostname) - if *debugFlag { - log.Printf("Box.SnapshotInitialize : %s : read %d zfs snapshots", b.Name, len(b.zfs.SnapshotList)) - } + b.online = true - b.zfs.SnapshotInitialized = true - b.zfs.SnapshotAdded = false - b.zfs.SnapshotDeleted = false + if err := b.zfs.Open(); err != nil { + log.WithFields(log.Fields{"name": b.name, "call": "zfs.Open", "error": err}).Errorf("") + return err + } return nil } -func (b *Box) ZFSUpdateList() (err error) { - if *debugFlag { - log.Printf("Box.ZFSUpdateList : %s : Start", b.Name) - } +func (b *Box) Close() error { + log.WithFields(log.Fields{"name": b.name}).Debugf("starting") + defer log.WithFields(log.Fields{"name": b.name}).Debugf("done") + + b.mx.Lock() + defer b.mx.Unlock() if !b.online { - err = fmt.Errorf("box offline") - return - } - - b.zfs.M.Lock() - if b.zfs.ZFSDeleted || b.zfs.ZFSAdded { - b.zfs.ZFSInitialized = false - } - b.zfs.M.Unlock() - - err = b.ZFSInitialize() - return -} - -func (b *Box) ZFSIsZFS(path string) bool { - if *debugFlag { - log.Printf("Box.ZFSIsZFS : %s : Start %s", b.Name, path) - } - - if !b.online { - return false - } - - err := b.ZFSInitialize() - if err != nil { - return false - } - - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - - if _, ok := b.zfs.ZFSMap[path]; ok { - return true - } - return false - -} - -func (b *Box) ZFSCreateZFS(path string) (err error) { - if *debugFlag { - log.Printf("Box.ZFSCreateZFS : %s : Start %s", b.Name, path) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - err = b.ZFSInitialize() - if err != nil { - return - } - - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - - p := strings.Split(path, `/`) - var base string - for _, d := range p { - if base == "" { - base = d - } else { - base = base + `/` + d - } - if _, ok := b.zfs.ZFSMap[base]; !ok { - - if *debugFlag { - log.Printf("Box.ZFSCreateZFS : Creating %s:%s", b.Name, base) - } - - _, err = b.SSHExec("zfs create -o mountpoint=none " + base) - if err != nil { - if *debugFlag { - log.Printf("Box.ZFSCreateZFS : %s : SSHExec : %s", b.Name, err) - } - return - } - - b.zfs.ZFSMap[base] = "none" - b.zfs.ZFSAdded = true - } - } - - return -} - -func (b *Box) ZFSInitialize() (err error) { - b.zfs.M.Lock() - defer b.zfs.M.Unlock() - if *debugFlag { - log.Printf("Box.ZFSInitialize : %s : Start", b.Name) - } - - if !b.online { - err = fmt.Errorf("box offline") - return - } - - if b.zfs.ZFSInitialized { return nil } - if *debugFlag { - log.Printf("Box.ZFSInitialize : %s : Start", b.Name) - } - - b.zfs.ZFSMap = make(map[string]string) - - var buf *bytes.Buffer - buf, err = b.SSHExec("zfs list -H -o name,mountpoint") - - csvReader := csv.NewReader(buf) - csvReader.Comma = '\t' - csvReader.FieldsPerRecord = 2 - - csvData, err := csvReader.ReadAll() - if err != nil { - if *debugFlag { - log.Printf("Box.ZFSInitialize : %s : csvReader.ReadAll() : %s", b.Name, err) - } + if err := b.zfs.Close(); err != nil { + log.WithFields(log.Fields{"name": b.name, "call": "zfs.Close", "error": err}).Errorf("") return err } - for _, rec := range csvData { - b.zfs.ZFSMap[rec[0]] = rec[1] - } - - b.zfs.ZFSInitialized = true - b.zfs.ZFSAdded = false - b.zfs.ZFSDeleted = false + b.online = false return nil } -func (b *Box) SSHExec(cmd string) (buf *bytes.Buffer, err error) { - if !b.online { - err = fmt.Errorf("box offline") - return +func (b *Box) Exec(cmd string) (r string, err error) { + log.WithFields(log.Fields{"name": b.name, "cmd": cmd}).Debugf("starting") + defer log.WithFields(log.Fields{"name": b.name, "cmd": cmd}).Debugf("done") + + if !b.created { + err := errors.New("box not initialized") + log.WithFields(log.Fields{"name": b.name, "error": err}).Errorf("") + return "", err } - buf, err = b.ssh.exec(cmd) - return + v, err := b.sshPool.Get() + if err != nil { + log.WithFields(log.Fields{"name": b.name, "error": err, "call": "SshPool.Get"}).Errorf("") + return "", err + } + + defer b.sshPool.Put(v) + s := v.(*Ssh) + + return s.Exec(cmd) } -func (b *Box) Host() string { - s := strings.Split(string(b.Addr), `:`) - return s[0] +func TransferZfs(from, to Addr) error { + log.WithFields(log.Fields{"from": from, "to": to}).Debugf("starting") + defer log.WithFields(log.Fields{"from": from, "to": to}).Debugf("done") + + var ( + err error + fromSnapshots, toSnapshots []*ZfsSnapshot + ) + + if fromSnapshots, err = from.ValidSnapshots(); err != nil { + log.WithFields(log.Fields{"from": from, "to": to, "call": "ValidSnapshots", "attr": from, "error": err}).Errorf("") + return err + } + + if len(fromSnapshots) == 0 { + return nil + } + + if toSnapshots, err = to.ValidSnapshots(); err != nil { + log.WithFields(log.Fields{"from": from, "to": to, "call": "ValidSnapshots", "attr": to, "error": err}).Errorf("") + return err + } + + if len(toSnapshots) == 0 { + log.WithFields(log.Fields{"from": from, "to": to}).Debugf("initiating destination") + if _, err := to.BoxExec("ssh " + from.Box() + " zfs send " + fromSnapshots[0].String() + " | zfs recv -F " + to.Path()); err != nil { + log.WithFields(log.Fields{"from": from, "to": to, "call": "BoxExec", "error": err}).Errorf("") + return err + } + newToSnapshot := &ZfsSnapshot{name: fromSnapshots[0].name, fs: cfg.box[to.Box()].zfs.filesystems[to.Path()]} + toSnapshots = append(toSnapshots, newToSnapshot) + cfg.box[to.Box()].zfs.filesystems[to.Path()].AddSnapshot(newToSnapshot) + } + + fromFromSnapshotId := -1 + lastToSnapshot := toSnapshots[len(toSnapshots)-1] + log.WithFields(log.Fields{"from": from, "to": to}).Debugf("searching last snapshot %s", lastToSnapshot.String()) + for id, v := range fromSnapshots { + if v.name == lastToSnapshot.name { + fromFromSnapshotId = id + log.WithFields(log.Fields{"from": from, "to": to}).Debugf("found %s", v.String()) + break + } + } + + if fromFromSnapshotId == -1 { + err := errors.New("zfs snapshot unsync") + log.WithFields(log.Fields{"from": from, "to": to, "error": err}).Errorf("") + return err + } + + if fromSnapshots[fromFromSnapshotId].name != lastToSnapshot.name { + log.WithFields(log.Fields{"from": from, "to": to}).Debugf("transfering from %s to %s", fromSnapshots[fromFromSnapshotId].name, fromSnapshots[len(fromSnapshots)-1].name) + + if _, err := to.BoxExec("ssh " + from.Box() + " zfs send -I " + fromSnapshots[fromFromSnapshotId].String() + " " + fromSnapshots[len(fromSnapshots)-1].String() + " | zfs recv -F " + to.Path()); err != nil { + log.WithFields(log.Fields{"from": from, "to": to, "call": "BoxExec", "error": err}).Errorf("") + return err + } + + for _, v := range fromSnapshots[fromFromSnapshotId:] { + cfg.box[to.Box()].zfs.filesystems[to.Path()].AddSnapshot(&ZfsSnapshot{name: v.name, fs: cfg.box[to.Box()].zfs.filesystems[to.Path()]}) + } + } + + return nil } diff --git a/config.go b/config.go index bc2c2fd..96c5865 100644 --- a/config.go +++ b/config.go @@ -1,227 +1,199 @@ package main import ( - "bytes" "encoding/json" + "errors" "fmt" "io/ioutil" - "log" - "regexp" + "sync" "time" - "golang.org/x/crypto/ssh" + log "github.com/sirupsen/logrus" ) type Config struct { - Zfsnap map[string]string `json:"zfsnap"` - Box map[string]*Box `json:"box"` - Email EmailConfig `json:"email"` - Apps []AppConfig `json:"apps"` - Timezone string `json:"timezone"` - Now time.Time `json:"-"` + ScheduleDuration map[string]string `json:"schedule"` + Box map[string]BoxConfig `json:"box"` + Email EmailConfig `json:"email"` + Apps []AppConfig `json:"apps"` + Timezone string `json:"timezone"` + Debug bool `json:"debug"` + Admin AdminConfig `json:"admin"` + box map[string]*Box `json:"-"` + apps map[string]*App `json:"-"` + timezone *time.Location `json:"-"` } -//Load config from file -func (c *Config) Load() error { - if *debugFlag { - log.Printf("SSHConfig.Load : Start") - } - b, err := ioutil.ReadFile(*cfgFile) +type AdminConfig struct { + Addr string `json:"addr"` + Username string `json:"username"` + Password string `json:"password"` +} + +type BoxConfig struct { + Addr string `json:"addr"` + User string `json:"user"` + Key string `json:"key"` +} + +type AppConfig struct { + Name string `json:"name"` + Schedule []string `json:"schedule"` + Sources []string `json:"src"` + Destinations []string `json:"dest"` + Before map[string]string `json:"before"` + After map[string]string `json:"after"` + Active bool `json:"active"` +} + +// Load config from file +func (c *Config) LoadFile(path string) error { + log.WithFields(log.Fields{"path": path}).Debugf("starting") + defer log.WithFields(log.Fields{"path": path}).Debugf("done") + + b, err := ioutil.ReadFile(path) if err != nil { - if *debugFlag { - log.Printf("Config.Load : ioutil.ReadFile(%s) : %s", *cfgFile, err) - } + log.WithFields(log.Fields{"path": path, "error": err, "call": "ioutil.ReadFile"}).Errorf("") return err } err = json.Unmarshal(b, &c) if err != nil { - if *debugFlag { - log.Printf("Config.Load : json.Unmarshal : %s", err) - } + log.WithFields(log.Fields{"path": path, "error": err, "call": "json.Unmarshal"}).Errorf("") return err } - if *debugFlag { - log.Printf("Config.Load :\r\n%v", cfg) - } - - l, err := time.LoadLocation(cfg.Timezone) + c.timezone, err = time.LoadLocation(cfg.Timezone) if err != nil { - if *debugFlag { - log.Printf("Config.Load : time.LoadLocation : %s", err) - } + log.WithFields(log.Fields{"path": path, "error": err, "call": "time.LoadLocation", "attr": cfg.Timezone}).Errorf("") return err } if len(cfg.Email.SmtpHost) == 0 { - if *debugFlag { - log.Printf("Config.Load : no smtp") - } - return fmt.Errorf("no smtp") + err := fmt.Errorf("no smtp") + log.WithFields(log.Fields{"path": path, "error": err}).Errorf("") + return err } if len(cfg.Email.FromEmail) == 0 { - if *debugFlag { - log.Printf("Config.Load : no email from") - } - return fmt.Errorf("no email from") + err := fmt.Errorf("no email from") + log.WithFields(log.Fields{"path": path, "error": err}).Errorf("") + return err } if len(cfg.Email.ToEmail) == 0 { - if *debugFlag { - log.Printf("Config.Load : no email to") - } - return fmt.Errorf("no email to") + err := fmt.Errorf("no email to") + log.WithFields(log.Fields{"path": path, "error": err}).Errorf("") + return err } - c.Now = time.Now().In(l) + for k, v := range c.ScheduleDuration { + switch k { + case "hourly": + case "daily": + case "weekly": + case "monthly": + case "yearly": + if _, err := Expiration(time.Now(), v); err != nil { + log.WithFields(log.Fields{"path": path, "schedule": k, "deadline": v, "error": err}).Errorf("") + return err + } + default: + err := errors.New("invalid schedule") + log.WithFields(log.Fields{"path": path, "schedule": k, "deadline": v, "error": err}).Errorf("") + return err + } + } + c.box = make(map[string]*Box) for k, v := range c.Box { - v.Name = k - v.online = false - v.zfs = NewZFSConfig() - s := &SSHConfig{ - logged: false, - name: k, - } - v.ssh = s - keyRaw, err := ioutil.ReadFile(v.Key) - if err != nil { - if *debugFlag { - log.Printf("Config.Load : ioutil.ReadFile(%s) : %s", k, err) - } + if b, err := c.NewBox(k, v.Addr, v.User, v.Key); err != nil { + log.WithFields(log.Fields{"path": path, "call": "NewBox", "attr": k, "error": err}).Errorf("") return err - } - - key, err := ssh.ParseRawPrivateKey(keyRaw) - if err != nil { - if *debugFlag { - log.Printf("Config.Load : ssh.ParseRawPrivateKey(%s) : %s", k, err) - } - return err - } - - s.signer, err = ssh.NewSignerFromKey(key) - if err != nil { - if *debugFlag { - log.Printf("Config.Load : ssh.NewSignerFromKey(%s) : %s", k, err) - } - return err - } - - s.config = &ssh.ClientConfig{ - User: v.User, - Auth: []ssh.AuthMethod{ - ssh.PublicKeys(s.signer), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - Timeout: 5 * time.Second, - } - - s.client, err = ssh.Dial("tcp", v.Addr, s.config) - if err != nil { - if *debugFlag { - log.Printf("Config.Load : ssh.Dial(%s) : %s", k, err) - } } else { - v.online = true - session, err := s.client.NewSession() - if err != nil { - if *debugFlag { - log.Printf("Config.Load : client.NewSession(%s) : %s", k, err) - } + if _, ok := c.box[k]; ok { + err := errors.New("already exists") + log.WithFields(log.Fields{"path": path, "attr": k, "error": err}).Errorf("") return err } - - var b bytes.Buffer - session.Stdout = &b - - err = session.Run("TZ=\"" + cfg.Timezone + "\" zfsnap --version") - if err != nil { - if *debugFlag { - log.Printf("Config.Load : client.NewSession(%s) : %s", k, err) - } - return err - } - if *debugFlag { - log.Printf("Config.Load : logged into %s : %s", k, b.String()) - } - session.Close() - s.logged = true + c.box[k] = b } } - for _, app := range c.Apps { - for _, src := range app.Sources { - if !src.Valid() { - return fmt.Errorf("Source not valid : %s", string(src)) - } - if _, ok := cfg.Box[src.Box()]; !ok { - return fmt.Errorf("No box defined for source : %s", string(src)) - } - if !cfg.Box[src.Box()].online { - email.items = append(email.items, fmt.Sprintf("Source box offline for app : %s", app.Name)) - } - } - var allOffline bool = true - for _, dest := range app.Destinations { - if !dest.Valid() { - return fmt.Errorf("Destination not valid : %s", string(dest)) - } - if _, ok := cfg.Box[dest.Box()]; !ok { - return fmt.Errorf("No box defined for destination : %s", string(dest)) - } - if cfg.Box[dest.Box()].online { - allOffline = false - } - } - if allOffline { - email.items = append(email.items, fmt.Sprintf("No online destination box for app : %s", app.Name)) - } - - for val, before := range app.Before { - _, err = regexp.Compile(val) - if err != nil { - if *debugFlag { - log.Printf("Config.Load : invalid regex : %s", val) - } + c.apps = make(map[string]*App) + for _, v := range c.Apps { + if a, err := c.NewApp(v.Name, v.Sources, v.Destinations, v.Schedule, v.Before, v.After); err != nil { + log.WithFields(log.Fields{"path": path, "call": "NewApp", "attr": v.Name, "error": err}).Errorf("") + return err + } else { + if _, ok := c.apps[v.Name]; ok { + err := errors.New("app already exists") + log.WithFields(log.Fields{"path": path, "app": v.Name, "error": err}).Errorf("") return err } - if !before.Valid() { - return fmt.Errorf("Before not valid : %s", string(before)) - } - if _, ok := cfg.Box[before.Box()]; !ok { - return fmt.Errorf("No box defined for before : %s", string(before)) - } - if !cfg.Box[before.Box()].online { - email.items = append(email.items, fmt.Sprintf("Before box offline for app : %s", app.Name)) - } - } - for val, after := range app.After { - _, err = regexp.Compile(val) - if err != nil { - if *debugFlag { - log.Printf("Config.Load : invalid regex : %s", val) - } - return err - } - if !after.Valid() { - return fmt.Errorf("After not valid : %s", string(after)) - } - if _, ok := cfg.Box[after.Box()]; !ok { - return fmt.Errorf("No box defined for after : %s", string(after)) - } - if !cfg.Box[after.Box()].online { - email.items = append(email.items, fmt.Sprintf("After box offline for app : %s", app.Name)) - } + c.apps[v.Name] = a } } return nil } -//Close config -func (c *Config) Close() error { - return nil +func (c *Config) Start(e *Email) { + log.WithFields(log.Fields{}).Debugf("starting") + defer log.WithFields(log.Fields{}).Debugf("done") + + var wg sync.WaitGroup + for _, b := range c.box { + wg.Add(1) + go func(box *Box) { + defer wg.Done() + if err := box.Open(); err != nil { + log.WithFields(log.Fields{"name": box.name, "call": "Open", "error": err}).Errorf("") + if e != nil { + e.AddItem(fmt.Sprintf(" - Box : %s is down", box.name)) + } + return + } + }(b) + } + + wg.Wait() +} + +// Run config +func (c *Config) Run(e *Email) { + log.WithFields(log.Fields{}).Debugf("starting") + defer log.WithFields(log.Fields{}).Debugf("done") + + var wg sync.WaitGroup + for _, a := range cfg.apps { + wg.Add(1) + go func(app *App) { + if err := app.Run(e.startTime); err != nil { + e.AddItem(fmt.Sprintf(" - App : Error running %s (%s)", app.name, err)) + } + wg.Done() + }(a) + } + + wg.Wait() + + return +} + +func (c *Config) Stop(e *Email) { + log.WithFields(log.Fields{}).Debugf("starting") + defer log.WithFields(log.Fields{}).Debugf("done") + + for _, b := range c.box { + if err := b.Close(); err != nil { + log.WithFields(log.Fields{"name": b.name, "call": "Close", "error": err}).Errorf("") + } + } + + if len(e.items) > 0 { + if err := e.Send(); err != nil { + log.WithFields(log.Fields{"call": "email.Send", "error": err}).Errorf("") + } + } } diff --git a/const.go b/const.go new file mode 100644 index 0000000..bc54137 --- /dev/null +++ b/const.go @@ -0,0 +1,13 @@ +package main + +const ( + boxNamePattern = `[a-zA-Z0-9\-_\.]` + + zfsManagedPropertyName = "biz.siteop:managed" + zfsSnapshotPattern = `^(?Phourly|daily|weekly|monthly|yearly|adhoc)\-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})\-\-(?Pforever|([0-9]+(h|d|m|y))+)$` + zfsSnapshotDatePattern = "2006-01-02_15.04.05" + + serverAddr = ":8080" + serverUsername = "admin" + serverPassword = "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918" //admin +) diff --git a/cron.go b/cron.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/cron.go @@ -0,0 +1 @@ +package main diff --git a/email.go b/email.go index a9d39a9..28af944 100644 --- a/email.go +++ b/email.go @@ -2,15 +2,19 @@ package main import ( "encoding/base64" - "log" + "fmt" "net/smtp" "strings" + "sync" "time" + + log "github.com/sirupsen/logrus" ) type Email struct { startTime time.Time items []string + mx sync.Mutex } type EmailConfig struct { @@ -19,44 +23,70 @@ type EmailConfig struct { ToEmail []string `json:"email_to"` } -func SendMail(addr, from, subject, body string, to []string) error { - if *debugFlag { - log.Printf("SendMail : Start") +func NewEmail(now time.Time) *Email { + log.WithFields(log.Fields{"now": now}).Debugf("starting") + defer log.WithFields(log.Fields{"now": now}).Debugf("done") + + return &Email{startTime: now, items: make([]string, 0)} +} + +func (e *Email) AddItem(item string) { + log.WithFields(log.Fields{"item": item}).Debugf("starting") + defer log.WithFields(log.Fields{"item": item}).Debugf("done") + + e.items = append(e.items, item) +} + +func (e *Email) Send() error { + log.WithFields(log.Fields{}).Debugf("starting") + defer log.WithFields(log.Fields{}).Debugf("done") + + body := " - " + e.items[0] + for _, item := range e.items[1:] { + body = body + "\r\n" + item } + + subject := fmt.Sprintf("Autobackup report (%s)", e.startTime) + + if err := SendMail(cfg.Email.SmtpHost, cfg.Email.FromEmail, subject, body, cfg.Email.ToEmail); err != nil { + log.WithFields(log.Fields{"addr": cfg.Email.SmtpHost, "from": cfg.Email.FromEmail, "subject": subject, "call": "SendMail", "error": err}).Errorf("") + return err + } + + return nil + +} + +func SendMail(addr, from, subject, body string, to []string) error { + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject}).Debugf("starting") + defer log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject}).Debugf("done") + r := strings.NewReplacer("\r\n", "", "\r", "", "\n", "", "%0a", "", "%0d", "") c, err := smtp.Dial(addr) if err != nil { - if *debugFlag { - log.Printf("SendMail : %s : smtp.Dial (%s)", addr, err) - } + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject, "call": "smtp.Dial", "error": err}).Errorf("") return err } defer c.Close() if err = c.Mail(r.Replace(from)); err != nil { - if *debugFlag { - log.Printf("SendMail : %s : client.Mail (%s)", from, err) - } + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject, "call": "client.Mail", "error": err}).Errorf("") return err } for i := range to { to[i] = r.Replace(to[i]) if err = c.Rcpt(to[i]); err != nil { - if *debugFlag { - log.Printf("SendMail : %s : client.Rcpt (%s)", to[i], err) - } + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject, "call": "client.Rcpt", "attr": to[i], "error": err}).Errorf("") return err } } w, err := c.Data() if err != nil { - if *debugFlag { - log.Printf("SendMail : client.Data (%s)", err) - } + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject, "call": "client.Date", "error": err}).Errorf("") return err } @@ -68,29 +98,21 @@ func SendMail(addr, from, subject, body string, to []string) error { "Content-Transfer-Encoding: base64\r\n" + "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) - if *debugFlag { - log.Printf("SendMail :\r\n%s", msg) - } - _, err = w.Write([]byte(msg)) if err != nil { - if *debugFlag { - log.Printf("SendMail : writer.Write (%s)", err) - } + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject, "call": "writer.Write", "error": err}).Errorf("") return err } err = w.Close() if err != nil { - if *debugFlag { - log.Printf("SendMail : writer.Close (%s)", err) - } + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject, "call": "writer.Close", "error": err}).Errorf("") return err } - err = c.Quit() - if *debugFlag { - log.Printf("SendMail : client.Quit (%s)", err) + if err = c.Quit(); err != nil { + log.WithFields(log.Fields{"addr": addr, "from": from, "subject": subject, "call": "client.Quit", "error": err}).Errorf("") + return err } - return err + return nil } diff --git a/go.mod b/go.mod index e65a10a..ecc3fab 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module git.siteop.biz/shoopea/backup go 1.16 require ( - golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b - golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect + github.com/silenceper/pool v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + golang.org/x/crypto v0.9.0 ) diff --git a/go.sum b/go.sum index 7c860cc..04f3e81 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,136 @@ +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/silenceper/pool v1.0.0 h1:JTCaA+U6hJAA0P8nCx+JfsRCHMwLTfatsm5QXelffmU= +github.com/silenceper/pool v1.0.0/go.mod h1:3DN13bqAbq86Lmzf6iUXWEPIWFPOSYVfaoceFvilKKI= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b h1:huxqepDufQpLLIRXiVkTvnxrzJlpwmIWAObmcCcUFr0= golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 h1:AzgQNqF+FKwyQ5LbVrVqOcuuFB67N47F9+htZYH0wFM= golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/location.go b/location.go index 3d8f8af..06ab7d0 100644 --- a/location.go +++ b/location.go @@ -1,20 +1 @@ package main - -import "strings" - -type Location string - -func (l Location) Box() string { - s := strings.Split(string(l), `:`) - return s[0] -} - -func (l Location) Path() string { - s := strings.Split(string(l), `:`) - return s[1] -} - -func (l Location) Valid() bool { - s := strings.Split(string(l), `:`) - return len(s) == 2 -} diff --git a/server.go b/server.go new file mode 100644 index 0000000..1534fc1 --- /dev/null +++ b/server.go @@ -0,0 +1,44 @@ +package main + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +type Server struct { + Addr string + Username string + Password string +} + +func NewServer(addr, username, password string) *Server { + s := &Server{ + Addr: serverAddr, + Username: serverUsername, + Password: serverPassword, + } + + if addr != "" { + s.Addr = addr + } + if username != "" { + s.Username = username + } + if password != "" { + s.Password = password + } + + return s + +} + +func (s *Server) Run() { + r := gin.Default() + r.GET("/ping", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "message": "pong", + }) + }) + r.Run(s.Addr) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") +} diff --git a/snapshot.go b/snapshot.go index a8f65aa..8e50480 100644 --- a/snapshot.go +++ b/snapshot.go @@ -1,20 +1,123 @@ package main -import "strings" +import ( + "errors" + "regexp" + "time" -type Snapshot string + log "github.com/sirupsen/logrus" +) -func (s Snapshot) Path() string { - s2 := strings.Split(string(s), `@`) - return s2[0] +func SnapshotName(schedule string, now time.Time) string { + log.WithFields(log.Fields{"schedule": schedule, "now": now}).Debugf("starting") + log.WithFields(log.Fields{"schedule": schedule, "now": now}).Debugf("done") + + return schedule + "-" + now.Format(zfsSnapshotDatePattern) + "--" + cfg.ScheduleDuration[schedule] } -func (s Snapshot) Name() string { - s2 := strings.Split(string(s), `@`) - return s2[1] +func (s *ZfsSnapshot) Valid() bool { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("done") + + re := regexp.MustCompile(zfsSnapshotPattern) + return re.MatchString(s.name) } -func (s Snapshot) Append(path string) Snapshot { - s2 := strings.Split(string(s), `@`) - return Snapshot(s2[0] + "/" + path + "@" + s2[1]) +func (s *ZfsSnapshot) Schedule() (string, error) { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("done") + + if !s.Valid() { + err := errors.New("invalid name") + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name, "call": "Valid", "error": err}).Errorf("") + return "", err + } + + re := regexp.MustCompile(zfsSnapshotPattern) + + return re.ReplaceAllString(s.name, "${Schedule}"), nil + +} + +func (s *ZfsSnapshot) Expiration() (time.Time, error) { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("done") + + if !s.Valid() { + err := errors.New("invalid name") + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name, "call": "Valid", "error": err}).Errorf("") + return time.Now(), err + } + + re := regexp.MustCompile(zfsSnapshotPattern) + + expirationString := re.ReplaceAllString(s.name, "${Expiration}") + + timestampTime, err := s.Timestamp() + if err != nil { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name, "call": "Timestamp", "error": err}).Errorf("") + return time.Now(), err + } + + return Expiration(timestampTime, expirationString) + +} + +func (s *ZfsSnapshot) Timestamp() (time.Time, error) { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("done") + + t := time.Now() + + if !s.Valid() { + err := errors.New("invalid name") + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name, "call": "Valid", "error": err}).Errorf("") + return t, err + } + + re := regexp.MustCompile(zfsSnapshotPattern) + + timestampString := re.ReplaceAllString(s.name, "${Timestamp}") + timestampTime, err := time.ParseInLocation(zfsSnapshotDatePattern, timestampString, cfg.timezone) + if err != nil { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name, "call": "time.Parse", "attr": timestampString, "error": err}).Errorf("") + return t, err + } + + return timestampTime, nil +} + +func (s *ZfsSnapshot) Expired(now time.Time) (bool, error) { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("done") + + if !s.Valid() { + err := errors.New("invalid name") + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name, "call": "Valid", "error": err}).Errorf("") + return false, err + } + + expirationTime, err := s.Expiration() + if err != nil { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name, "call": "Timestamp", "error": err}).Errorf("") + return false, err + } + + if now.After(expirationTime) { + return true, nil + } else { + return false, nil + } + +} + +func (s *ZfsSnapshot) String() string { + return s.fs.path + "@" + s.name +} + +func (s *ZfsSnapshot) Delete() error { + log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": s.fs.zfs.box.name, "fs": s.fs.path, "name": s.name}).Debugf("done") + + return s.fs.DelSnapshot(s.name) } diff --git a/ssh.go b/ssh.go index 2438ff4..009da3e 100644 --- a/ssh.go +++ b/ssh.go @@ -2,46 +2,121 @@ package main import ( "bytes" - "log" + "os" + "time" + "github.com/silenceper/pool" + log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" ) -type SSHConfig struct { - signer ssh.Signer - config *ssh.ClientConfig - client *ssh.Client - logged bool - name string - snapshot []Snapshot +const SshDialTimeout = time.Duration(10 * time.Second) +const SshInactivityTimeout = time.Duration(time.Minute) + +type Ssh struct { + name string + signer ssh.Signer + config *ssh.ClientConfig + client *ssh.Client } -func (s *SSHConfig) exec(cmd string) (b *bytes.Buffer, err error) { - if *debugFlag { - log.Printf("SSHConfig.exec : %s : Start %s", s.name, cmd) +func NewSsh(name, addr, user, key string) (*Ssh, error) { + log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key}).Debugf("starting") + defer log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key}).Debugf("done") + + s := &Ssh{ + name: name, } + k, err := os.ReadFile(key) + if err != nil { + log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key, "call": "os.ReadFile", "error": err}).Errorf("") + return s, err + } + + parsedKey, err := ssh.ParseRawPrivateKey(k) + if err != nil { + log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key, "call": "ssh.ParseRawPrivateKey", "error": err}).Errorf("") + return s, err + } + + s.signer, err = ssh.NewSignerFromKey(parsedKey) + if err != nil { + log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key, "call": "ssh.NewSignerFromKey", "error": err}).Errorf("") + return s, err + } + + s.config = &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(s.signer), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: SshDialTimeout, + } + + s.client, err = ssh.Dial("tcp", addr, s.config) + if err != nil { + log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key, "call": "ssh.Dial", "error": err}).Errorf("") + return s, err + } + + return s, nil + +} + +func (s *Ssh) Close() error { + log.WithFields(log.Fields{"name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"name": s.name}).Debugf("done") + + return s.client.Close() +} + +func NewSshPool(name, addr, user, key string) (pool.Pool, error) { + log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key}).Debugf("starting") + defer log.WithFields(log.Fields{"name": name, "addr": addr, "user": user, "key": key}).Debugf("done") + + //factory Specify the method to create the connection + factory := func() (interface{}, error) { return NewSsh(name, addr, user, key) } + + // close Specify the method to close the connection + close := func(v interface{}) error { return v.(*Ssh).Close() } + + // Create a connection pool: Initialize the number of connections to 0, the maximum idle connection is 2, and the maximum concurrent connection is 25 + poolConfig := &pool.Config{ + InitialCap: 0, + MaxIdle: 2, + MaxCap: 25, + Factory: factory, + Close: close, + //Ping: ping, + //The maximum idle time of the connection, the connection exceeding this time will be closed, which can avoid the problem of automatic failure when connecting to EOF when idle + IdleTimeout: SshInactivityTimeout, + } + + return pool.NewChannelPool(poolConfig) +} + +func (s *Ssh) Exec(cmd string) (string, error) { + log.WithFields(log.Fields{"name": s.name, "cmd": cmd}).Debugf("starting") + defer log.WithFields(log.Fields{"name": s.name, "cmd": cmd}).Debugf("done") + session, err := s.client.NewSession() if err != nil { - if *debugFlag { - log.Printf("SSHConfig.exec : %s : client().NewSession(%s) : %s", s.name, cmd, err) - } - return + log.WithFields(log.Fields{"name": s.name, "cmd": cmd, "call": "client.NewSession", "error": err}).Errorf("") + return "", err } + defer session.Close() - var buf bytes.Buffer - b = &buf - session.Stdout = b + var bufout, buferr bytes.Buffer + session.Stdout = &bufout + session.Stderr = &buferr err = session.Run("TZ=\"" + cfg.Timezone + "\" " + cmd) if err != nil { - if *debugFlag { - log.Printf("SSHConfig.exec : session(%s).Run(%s) : %s", s.name, cmd, err) - } - return + log.WithFields(log.Fields{"name": s.name, "cmd": cmd, "call": "session.Run", "error": err, "stderr": buferr.String()}).Errorf("") + return "", err } - session.Close() - - return + return bufout.String(), nil } diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..7e8ac26 --- /dev/null +++ b/utils.go @@ -0,0 +1,41 @@ +package main + +import ( + "errors" + "regexp" + "strconv" + "time" + + log "github.com/sirupsen/logrus" +) + +func Expiration(now time.Time, deadline string) (time.Time, error) { + log.WithFields(log.Fields{"now": now, "deadline": deadline}).Debugf("starting") + defer log.WithFields(log.Fields{"now": now, "deadline": deadline}).Debugf("done") + + if deadline == "forever" { + return time.Unix(1<<63-1, 0), nil + } + + reExpiration := regexp.MustCompile(`([0-9]+)([a-z]+)`) + for _, v := range reExpiration.FindAllStringSubmatch(deadline, -1) { + log.WithFields(log.Fields{"now": now, "deadline": deadline}).Debugf("duration[%d] : %v", len(v), v) + count, _ := strconv.Atoi(v[1]) + switch v[2] { + case "y": + now = now.AddDate(count, 0, 0) + case "m": + now = now.AddDate(0, count, 0) + case "d": + now = now.AddDate(0, 0, count) + case "h": + now = now.Add(time.Duration(time.Duration(count) * time.Hour)) + default: + err := errors.New("invalid duration") + log.WithFields(log.Fields{"now": now, "deadline": deadline, "attr": v[2], "error": err}).Errorf("") + return time.Now(), err + } + } + + return now, nil +} diff --git a/version.go b/version.go index d506661..0ffff19 100644 --- a/version.go +++ b/version.go @@ -1,6 +1,7 @@ // Code generated by version.sh (@generated) DO NOT EDIT. package main -var githash = "7f9cf49" -var buildstamp = "2022-10-08_03:14:52" -var commits = "54" -var version = "7f9cf49-b54 - 2022-10-08_03:14:52" +var githash = "fb02d52" +var branch = "v2" +var buildstamp = "2023-06-29_20:53:51" +var commits = "55" +var version = "fb02d52-b55 - 2023-06-29_20:53:51" diff --git a/version.sh b/version.sh index a436a5c..997db9e 100644 --- a/version.sh +++ b/version.sh @@ -1,5 +1,6 @@ # Get the version. githash=`git rev-parse --short HEAD` +branch=`git rev-parse --abbrev-ref HEAD` buildstamp=`date -u '+%Y-%m-%d_%H:%M:%S'` commits=`git rev-list --count master` # Write out the package. @@ -7,6 +8,7 @@ cat << EOF > version.go // Code generated by version.sh (@generated) DO NOT EDIT. package main var githash = "$githash" +var branch = "$branch" var buildstamp = "$buildstamp" var commits = "$commits" var version = "$githash-b$commits - $buildstamp" diff --git a/zfs.go b/zfs.go index 6f21a0e..611f702 100644 --- a/zfs.go +++ b/zfs.go @@ -1,27 +1,328 @@ package main -import "sync" +import ( + "bytes" + "encoding/csv" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "sync" -type ZFSConfig struct { - SnapshotAdded bool - SnapshotDeleted bool - SnapshotInitialized bool - SnapshotList []Snapshot - ZFSAdded bool - ZFSDeleted bool - ZFSInitialized bool - ZFSMap map[string]string - M sync.Mutex + log "github.com/sirupsen/logrus" +) + +type BoxZfs struct { + filesystems map[string]*ZfsFs + box *Box + online bool + mx sync.Mutex } -func NewZFSConfig() (z *ZFSConfig) { - z = &ZFSConfig{ - SnapshotAdded: false, - SnapshotDeleted: false, - SnapshotInitialized: false, - ZFSAdded: false, - ZFSDeleted: false, - ZFSInitialized: false, +type ZfsFs struct { + path string + managed bool + zfs *BoxZfs + snapshots map[string]*ZfsSnapshot + apps []*App + mx sync.Mutex +} + +type ZfsSnapshot struct { + name string + fs *ZfsFs +} + +func (z *BoxZfs) Open() error { + log.WithFields(log.Fields{"name": z.box.name}).Debugf("starting") + defer log.WithFields(log.Fields{"name": z.box.name}).Debugf("done") + + z.mx.Lock() + defer z.mx.Unlock() + + if z.online { + return nil } - return + + z.filesystems = make(map[string]*ZfsFs) + + zfsList, err := z.box.Exec("zfs list -H -t filesystem -o name,mountpoint") + if err != nil { + log.WithFields(log.Fields{"name": z.box.name, "call": "Exec", "attr": "zfs list -H -t filesystem -o name,mountpoint", "error": err}).Errorf("") + return err + } + + csvReader := csv.NewReader(bytes.NewBufferString(zfsList)) + csvReader.Comma = '\t' + csvReader.FieldsPerRecord = 2 + + csvData, err := csvReader.ReadAll() + if err != nil { + log.WithFields(log.Fields{"name": z.box.name, "call": "csvReader.ReadAll", "error": err}).Errorf("") + return err + } + + for _, rec := range csvData { + log.WithFields(log.Fields{"name": z.box.name, "zfs-name": rec[0], "zfs-mount": rec[1]}).Debugf("zfs list -t filesystem") + if rec[1] != "legacy" { + fs := &ZfsFs{ + path: rec[0], + zfs: z, + snapshots: make(map[string]*ZfsSnapshot), + } + z.filesystems[rec[0]] = fs + log.WithFields(log.Fields{"name": z.box.name, "fs": rec[0]}).Infof("new filesystem") + } + } + + log.WithFields(log.Fields{"name": z.box.name, "call": "csvReader.ReadAll"}).Infof("") + + zfsList, err = z.box.Exec("zfs list -H -t snapshot -o name") + if err != nil { + log.WithFields(log.Fields{"name": z.box.name, "call": "Exec", "attr": "zfs list -H -t snapshot -o name", "error": err}).Errorf("") + return err + } + + csvReader = csv.NewReader(bytes.NewBufferString(zfsList)) + csvReader.Comma = '\t' + csvReader.FieldsPerRecord = 1 + + csvData, err = csvReader.ReadAll() + if err != nil { + log.WithFields(log.Fields{"name": z.box.name, "call": "csvReader.ReadAll", "error": err}).Errorf("") + return err + } + + for _, rec := range csvData { + log.WithFields(log.Fields{"name": z.box.name, "zfs-snapshot": rec[0]}).Debugf("zfs list -t snapshot") + + s := strings.Split(rec[0], `@`) + if fs, ok := z.filesystems[s[0]]; ok { + snap := &ZfsSnapshot{ + name: s[1], + fs: fs, + } + fs.snapshots[s[1]] = snap + log.WithFields(log.Fields{"name": z.box.name, "fs": s[0], "snapshot": s[1]}).Infof("new snapshot") + } + } + + zfsList, err = z.box.Exec("zfs get -H -o name,value " + zfsManagedPropertyName) + if err != nil { + log.WithFields(log.Fields{"name": z.box.name, "call": "Exec", "attr": "zfs get -H -o name,value,source " + zfsManagedPropertyName, "error": err}).Errorf("") + return err + } + + csvReader = csv.NewReader(bytes.NewBufferString(zfsList)) + csvReader.Comma = '\t' + csvReader.FieldsPerRecord = 2 + + csvData, err = csvReader.ReadAll() + if err != nil { + log.WithFields(log.Fields{"name": z.box.name, "call": "csvReader.ReadAll", "error": err}).Errorf("") + return err + } + + for _, rec := range csvData { + log.WithFields(log.Fields{"name": z.box.name, "zfs-fs": rec[0], "zfs-value": rec[1]}).Debugf("zfs get " + zfsManagedPropertyName) + + if fs, ok := z.filesystems[rec[0]]; ok { + if rec[1] == "true" { + fs.managed = true + log.WithFields(log.Fields{"name": z.box.name, "zfs-fs": rec[0], "zfs-value": rec[1]}).Infof("managed fs") + } + } + } + + z.online = true + + return nil +} + +func (z *BoxZfs) Close() error { + log.WithFields(log.Fields{"name": z.box.name}).Debugf("starting") + defer log.WithFields(log.Fields{"name": z.box.name}).Debugf("done") + + z.mx.Lock() + defer z.mx.Unlock() + + for _, fs := range z.filesystems { + fs.mx.Lock() + defer fs.mx.Unlock() + } + + z.online = false + + return nil +} + +func (z *BoxZfs) Mkdir(path string) error { + log.WithFields(log.Fields{"name": z.box.name}).Debugf("starting") + defer log.WithFields(log.Fields{"name": z.box.name}).Debugf("done") + + if !z.online { + err := errors.New("zfs offline") + log.WithFields(log.Fields{"name": z.box.name, "error": err}).Errorf("") + return err + } + + z.mx.Lock() + defer z.mx.Unlock() + + b := z.box + if !b.online { + err := errors.New("box offline") + log.WithFields(log.Fields{"name": z.box.name, "error": err}).Errorf("") + return err + } + + if _, ok := z.filesystems[path]; ok { + return nil + } + + if _, err := b.Exec(fmt.Sprintf("zfs create -p %s", path)); err != nil { + log.WithFields(log.Fields{"name": z.box.name, "call": "Exec", "error": err}).Errorf("") + return err + } + + newPath := "" + for _, p := range strings.Split(path, "/") { + if newPath == "" { + newPath = p + } else { + newPath = newPath + "/" + p + } + + if _, ok := z.filesystems[newPath]; !ok { + fs := &ZfsFs{ + path: newPath, + managed: false, + zfs: z, + snapshots: make(map[string]*ZfsSnapshot), + apps: make([]*App, 0), + } + z.filesystems[newPath] = fs + } + + } + + return nil +} + +func (fs *ZfsFs) TakeSnapshot(name string) (*ZfsSnapshot, error) { + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name}).Debugf("done") + + if !fs.zfs.online { + err := errors.New("zfs offline") + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name, "error": err}).Errorf("") + return nil, err + } + + re := regexp.MustCompile(`^[a-zA-Z0-9\-\._]{1,255}$`) + if !re.MatchString(name) { + err := errors.New("unsupported name") + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name, "error": err}).Errorf("") + return nil, err + } + + fs.mx.Lock() + defer fs.mx.Unlock() + + if _, ok := fs.snapshots[name]; ok { + err := errors.New("already exists") + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name, "error": err}).Errorf("") + return nil, err + } + + if _, err := fs.zfs.box.Exec("zfs snapshot " + fs.path + "@" + name); err != nil { + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name, "error": err}).Errorf("") + return nil, err + } + + s := &ZfsSnapshot{ + name: name, + fs: fs, + } + + fs.snapshots[name] = s + + return s, nil +} + +func (fs *ZfsFs) DelSnapshot(name string) error { + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name}).Debugf("done") + + if !fs.zfs.online { + err := errors.New("zfs offline") + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name, "error": err}).Errorf("") + return err + } + + fs.mx.Lock() + defer fs.mx.Unlock() + + if _, ok := fs.snapshots[name]; !ok { + err := errors.New("doesn't exist") + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name, "error": err}).Errorf("") + return err + } + + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name}).Debugf("zfs destroy " + fs.path + "@" + name) + + return nil + + if _, err := fs.zfs.box.Exec("zfs destroy " + fs.path + "@" + name); err != nil { + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": name, "error": err}).Errorf("") + return err + } + + delete(fs.snapshots, name) + + return nil + +} + +func (fs *ZfsFs) AddSnapshot(s *ZfsSnapshot) error { + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": s.name}).Debugf("starting") + defer log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": s.name}).Debugf("done") + + if !fs.zfs.online { + err := errors.New("zfs offline") + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": s.name, "error": err}).Errorf("") + return err + } + + fs.mx.Lock() + defer fs.mx.Unlock() + + if _, ok := fs.snapshots[s.name]; ok { + err := errors.New("already exist") + log.WithFields(log.Fields{"box": fs.zfs.box.name, "fs": fs.path, "name": s.name, "error": err}).Errorf("") + return err + } + + fs.snapshots[s.name] = s + + return nil +} + +func (fs *ZfsFs) ValidSnapshots() []*ZfsSnapshot { + tab := make([]*ZfsSnapshot, 0) + + for _, s := range fs.snapshots { + if s.Valid() { + tab = append(tab, s) + } + } + + sort.Slice(tab, func(i, j int) bool { + ti, _ := tab[i].Timestamp() + tj, _ := tab[j].Timestamp() + return ti.Before(tj) + }) + + return tab }