package main import ( "fmt" "log" "regexp" "time" ) type AppConfig struct { Name string `json:"name"` Schedule []string `json:"schedule"` Sources []Location `json:"src"` Destinations []Location `json:"dest"` Before map[string]Location `json:"before"` After map[string]Location `json:"after"` } func (a AppConfig) getSchedule() (schedule string, err error) { if *debugFlag { log.Printf("AppConfig.getSchedule : %s : Start", a.Name) } refreshSnapshot := make(map[string]bool) for _, v := range a.Sources { refreshSnapshot[v.Box()] = true } for k, _ := range refreshSnapshot { err := cfg.Box[k].ZFSUpdateSnapshotList() if err != nil { if *debugFlag { log.Printf("AppConfig.getSchedule : %s : ZFSUpdateSnapshotList(%s) : %s", a.Name, k, err) } return "", err } } var ok bool if *schedFlag != "" { schedule = *schedFlag } else if ok, err = a.needYearlySnapshot(); ok && err == nil { schedule = "yearly" } else if ok, err = a.needMonthlySnapshot(); ok && err == nil { schedule = "monthly" } else if ok, err = a.needWeeklySnapshot(); ok && err == nil { schedule = "weekly" } else if ok, err = a.needDailySnapshot(); ok && err == nil { schedule = "daily" } else if ok, err = a.needHourlySnapshot(); ok && err == nil { schedule = "hourly" } else { return } if ret, ok := cfg.Zfsnap[schedule]; !ok { schedule = "" err = fmt.Errorf("no retention for %s", schedule) } else { re := regexp.MustCompile(`^([0-9]+[ymwdhMs]{1}|forever)$`) if !re.MatchString(ret) { schedule = "" err = fmt.Errorf("wrong retention format for %s", schedule) } } return } func (a AppConfig) needYearlySnapshot() (ret bool, err error) { if *debugFlag { log.Printf("AppConfig.needYearlySnapshot : %s : Start", a.Name) } ret = false // schedule enabled for app ? for _, v := range a.Schedule { if v == "yearly" { ret = true } } if !ret { return } // finding out the timestamps existing timeSource := make(map[string]map[time.Time]struct{}) timeTotal := make(map[time.Time]struct{}) re := regexp.MustCompile(`^yearly-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) for _, src := range a.Sources { timeSource[string(src)] = make(map[time.Time]struct{}) var snapList []Snapshot snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() if err != nil { return } for _, snap := range snapList { if src.Path() == snap.Path() { if re.MatchString(snap.Name()) { dateString := re.ReplaceAllString(snap.Name(), "${Date}") var dateTime time.Time dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) if err != nil { if *debugFlag { log.Printf("AppConfig.needYearlySnapshot : %s : time.ParseInLocation(%s) : %s", a.Name, dateString, err) } return } else { timeSource[string(src)][dateTime] = struct{}{} timeTotal[dateTime] = struct{}{} } } } } } // cleaning up the available timestamps for common timestamps for t, _ := range timeTotal { for _, v := range timeSource { if _, ok := v[t]; !ok { delete(timeTotal, t) } } } // finding an eligible timestamp for t, _ := range timeTotal { if t.Year() == cfg.Now.Year() { ret = false } } // no timestamp => need the snapshot ! return } func (a AppConfig) needMonthlySnapshot() (ret bool, err error) { if *debugFlag { log.Printf("AppConfig.needMonthlySnapshot : %s : Start", a.Name) } ret = false // schedule enabled for app ? for _, v := range a.Schedule { if v == "monthly" { ret = true } } if !ret { return } // finding out the timestamps existing timeSource := make(map[string]map[time.Time]struct{}) timeTotal := make(map[time.Time]struct{}) re := regexp.MustCompile(`^(yearly|monthly)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) for _, src := range a.Sources { timeSource[string(src)] = make(map[time.Time]struct{}) var snapList []Snapshot snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() if err != nil { return } for _, snap := range snapList { if src.Path() == snap.Path() { if re.MatchString(snap.Name()) { dateString := re.ReplaceAllString(snap.Name(), "${Date}") var dateTime time.Time dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) if err != nil { if *debugFlag { log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) } return } else { timeSource[string(src)][dateTime] = struct{}{} timeTotal[dateTime] = struct{}{} } } } } } // cleaning up the available timestamps for common timestamps for t, _ := range timeTotal { for _, v := range timeSource { if _, ok := v[t]; !ok { delete(timeTotal, t) } } } // finding an eligible timestamp for t, _ := range timeTotal { if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() { ret = false } } // no timestamp => need the snapshot ! return } func (a AppConfig) needWeeklySnapshot() (ret bool, err error) { if *debugFlag { log.Printf("AppConfig.needWeeklySnapshot : %s : Start", a.Name) } ret = false // schedule enabled for app ? for _, v := range a.Schedule { if v == "weekly" { ret = true } } if !ret { return } // finding out the timestamps existing timeSource := make(map[string]map[time.Time]struct{}) timeTotal := make(map[time.Time]struct{}) re := regexp.MustCompile(`^(yearly|monthly|weekly)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) for _, src := range a.Sources { timeSource[string(src)] = make(map[time.Time]struct{}) var snapList []Snapshot snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() if err != nil { return } for _, snap := range snapList { if src.Path() == snap.Path() { if re.MatchString(snap.Name()) { dateString := re.ReplaceAllString(snap.Name(), "${Date}") var dateTime time.Time dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) if err != nil { if *debugFlag { log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) } return } else { timeSource[string(src)][dateTime] = struct{}{} timeTotal[dateTime] = struct{}{} } } } } } // cleaning up the available timestamps for common timestamps for t, _ := range timeTotal { for _, v := range timeSource { if _, ok := v[t]; !ok { delete(timeTotal, t) } } } // finding an eligible timestamp nowYear, nowWeek := cfg.Now.ISOWeek() for t, _ := range timeTotal { snapYear, snapWeek := t.ISOWeek() if nowYear == snapYear && nowWeek == snapWeek { ret = false } } // no timestamp => need the snapshot ! return } func (a AppConfig) needDailySnapshot() (ret bool, err error) { if *debugFlag { log.Printf("AppConfig.needDailySnapshot : %s : Start", a.Name) } ret = false // schedule enabled for app ? for _, v := range a.Schedule { if v == "daily" { ret = true } } if !ret { return } // finding out the timestamps existing timeSource := make(map[string]map[time.Time]struct{}) timeTotal := make(map[time.Time]struct{}) re := regexp.MustCompile(`^(yearly|monthly|weekly|daily)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) for _, src := range a.Sources { timeSource[string(src)] = make(map[time.Time]struct{}) var snapList []Snapshot snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() if err != nil { return } for _, snap := range snapList { if src.Path() == snap.Path() { if re.MatchString(snap.Name()) { dateString := re.ReplaceAllString(snap.Name(), "${Date}") var dateTime time.Time dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) if err != nil { if *debugFlag { log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) } return } else { timeSource[string(src)][dateTime] = struct{}{} timeTotal[dateTime] = struct{}{} } } } } } // cleaning up the available timestamps for common timestamps for t, _ := range timeTotal { for _, v := range timeSource { if _, ok := v[t]; !ok { delete(timeTotal, t) } } } // finding an eligible timestamp for t, _ := range timeTotal { if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() { ret = false } } // no timestamp => need the snapshot ! return } func (a AppConfig) needHourlySnapshot() (ret bool, err error) { if *debugFlag { log.Printf("AppConfig.needHourlySnapshot : %s : Start", a.Name) } ret = false // schedule enabled for app ? for _, v := range a.Schedule { if v == "hourly" { ret = true } } if !ret { return } // finding out the timestamps existing timeSource := make(map[string]map[time.Time]struct{}) timeTotal := make(map[time.Time]struct{}) re := regexp.MustCompile(`^(yearly|monthly|weekly|daily|hourly)-(?P[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) for _, src := range a.Sources { timeSource[string(src)] = make(map[time.Time]struct{}) var snapList []Snapshot snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList() if err != nil { return } for _, snap := range snapList { if src.Path() == snap.Path() { if re.MatchString(snap.Name()) { dateString := re.ReplaceAllString(snap.Name(), "${Date}") var dateTime time.Time dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location()) if err != nil { if *debugFlag { log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) } return } else { timeSource[string(src)][dateTime] = struct{}{} timeTotal[dateTime] = struct{}{} } } } } } // cleaning up the available timestamps for common timestamps for t, _ := range timeTotal { for _, v := range timeSource { if _, ok := v[t]; !ok { delete(timeTotal, t) } } } // finding an eligible timestamp for t, _ := range timeTotal { if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() && t.Hour() == cfg.Now.Hour() { ret = false } } // no timestamp => need the snapshot ! return } func (a AppConfig) CheckZFS() error { if *debugFlag { log.Printf("AppConfig.CheckZFS : %s : Start", a.Name) } for _, src := range a.Sources { if !cfg.Box[src.Box()].ZFSIsZFS(src.Path()) { return fmt.Errorf("No path %s on source", string(src)) } for _, dest := range a.Destinations { if !cfg.Box[dest.Box()].ZFSIsZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()) { err := cfg.Box[dest.Box()].ZFSCreateZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.CheckZFS : %s : Error creating %s on %s", a.Name, dest.Path()+"/"+src.Box()+"/"+src.Path(), dest.Box()) } return err } } } } return nil } func (a AppConfig) ExecBefore(schedule string) error { if *debugFlag { log.Printf("AppConfig.ExecBefore : %s : Start %s", a.Name, schedule) } for k, v := range a.Before { re := regexp.MustCompile(k) if re.MatchString(schedule) { _, err := cfg.Box[v.Box()].SSHExec(v.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.ExecBefore : %s : Error executing %s", a.Name, string(v)) } return err } } } return nil } func (a AppConfig) ExecAfter(schedule string) error { if *debugFlag { log.Printf("AppConfig.ExecAfter : %s : Start %s", a.Name, schedule) } for k, v := range a.After { re := regexp.MustCompile(k) if re.MatchString(schedule) { _, err := cfg.Box[v.Box()].SSHExec(v.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.ExecAfter : %s : Error executing %s on %s", a.Name, v.Path(), v.Box()) } return err } } } return nil } func (a AppConfig) TakeSnapshot(schedule string) error { if *debugFlag { log.Printf("AppConfig.TakeSnapshot : %s : Start %s", a.Name, schedule) } for _, v := range a.Sources { err := cfg.Box[v.Box()].ZFSTakeSnapshot(schedule, v.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.TakeSnapshot : %s : ZFSTakeSnapshot", a.Name) } return err } } return nil } func (a AppConfig) RefreshSnapshot() error { if *debugFlag { log.Printf("AppConfig.RefreshSnapshot : %s : Start", a.Name) } refreshSnapshot := make(map[string]struct{}) for _, v := range a.Sources { refreshSnapshot[v.Box()] = struct{}{} } for _, v := range a.Destinations { refreshSnapshot[v.Box()] = struct{}{} } for k, _ := range refreshSnapshot { if *debugFlag { log.Printf("AppConfig.RefreshSnapshot : %s : refreshing snapshots for source %s", a.Name, k) } err := cfg.Box[k].ZFSUpdateSnapshotList() if err != nil { if *debugFlag { log.Printf("AppConfig.RefreshSnapshot : %s : Error getting snapshots on %s", a.Name, k) } return err } } return nil } func (a AppConfig) SendSnapshots() (err error) { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Start", a.Name) } for _, src := range a.Sources { for _, dest := range a.Destinations { if cfg.Box[dest.Box()].online { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Sending snapshots from %s to %s", a.Name, string(src), string(dest)) } var dLastSnapshot Snapshot dLastSnapshot, err = cfg.Box[dest.Box()].ZFSGetLastSnapshot(dest.Path() + "/" + src.Box() + "/" + src.Path()) if err != nil && err.Error() == "no snapshot" { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s on %s", a.Name, string(src), dest.Box()) } var sFirstSnapshot Snapshot sFirstSnapshot, err = cfg.Box[src.Box()].ZFSGetFirstSnapshot(src.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s", a.Name, string(src)) } return } if !cfg.Box[dest.Box()].ZFSIsZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()) { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Creating on %s path %s", a.Name, dest.Box(), dest.Path()+"/"+src.Box()+"/"+src.Path()) } if err = cfg.Box[dest.Box()].ZFSCreateZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()); err != nil { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Creating on %s path %s failed (%s)", a.Name, dest.Box(), dest.Path()+"/"+src.Box()+"/"+src.Path(), err) } return } } if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s", a.Name, dest.Box(), string(sFirstSnapshot)) } _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send " + string(sFirstSnapshot) + " | zfs recv -F " + dest.Path() + "/" + src.Box() + "/" + src.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sFirstSnapshot), err) } return } var ( sCurrSnapshot, sNextSnapshot Snapshot isLastSnapshot bool ) sNextSnapshot = sFirstSnapshot isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) for !isLastSnapshot { sCurrSnapshot = sNextSnapshot sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot) if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box()) } if err != nil && err.Error() != "no snapshot" { return } _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err) } return } isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) } if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : All snapshots sent for %s", a.Name, string(src)) } } else { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Last snapshot on %s is %s", a.Name, dest.Box(), string(dLastSnapshot)) } var ( sCurrSnapshot, sNextSnapshot Snapshot isLastSnapshot bool ) sNextSnapshot = Snapshot(string(dLastSnapshot)[len(dest.Path())+len(src.Box())+2:]) isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) for !isLastSnapshot { sCurrSnapshot = sNextSnapshot sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot) if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box()) } if err != nil && err.Error() != "no snapshot" { return } _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path()) if err != nil { if *debugFlag { log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err) } return } isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) } } } } } err = nil return } func (a AppConfig) CleanupSnapshot() error { if *debugFlag { log.Printf("AppConfig.CleanupSnapshot : %s : Start", a.Name) } cleanupSnapshot := make(map[string]string) for _, dest := range a.Destinations { for _, src := range a.Sources { cleanupSnapshot[src.Box()] = cleanupSnapshot[src.Box()] + " " + src.Path() cleanupSnapshot[dest.Box()] = cleanupSnapshot[dest.Box()] + " " + dest.Path() + "/" + src.Box() + "/" + src.Path() } } for k, v := range cleanupSnapshot { if *debugFlag { log.Printf("AppConfig.CleanupSnapshot : %s : cleaning snapshots on %s for%s", a.Name, k, v) } _, err := cfg.Box[k].SSHExec("zfsnap destroy -p hourly- -p daily- -p weekly- -p monthly- -p yearly-" + v) if err != nil { if *debugFlag { log.Printf("AppConfig.CleanupSnapshot : %s : Error executing zfsnap on %s", a.Name, k) } return err } } return nil } func (a AppConfig) RunAppBackup() error { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : Start", a.Name) } schedule, err := a.getSchedule() if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : Error getting schedule : %s", a.Name, err) } return err } if schedule != "" || *slowFlag { err = a.CheckZFS() if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : CheckZFS : %s", a.Name, err) } return err } } if schedule != "" { err = a.ExecBefore(schedule) if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : ExecBefore : %s", a.Name, err) } return err } err = a.TakeSnapshot(schedule) if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : TakeSnapshot : %s", a.Name, err) } return err } } err = a.RefreshSnapshot() if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : RefreshSnapshot : %s", a.Name, err) } return err } err = a.SendSnapshots() if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : SendSnapshots : %s", a.Name, err) } return err } if schedule != "" { err = a.ExecAfter(schedule) if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : ExecAfter : %s", a.Name, err) } return err } } err = a.CleanupSnapshot() if err != nil { if *debugFlag { log.Printf("AppConfig.RunAppBackup : %s : RefreshSnapshot : %s", a.Name, err) } return err } return nil }