update snapshot processing

This commit is contained in:
shoopea 2021-11-14 17:20:44 +08:00
parent e192906265
commit 0d3981ef4a
4 changed files with 289 additions and 204 deletions

202
app.go
View File

@ -16,8 +16,7 @@ type AppConfig struct {
After map[string]Location `json:"after"` After map[string]Location `json:"after"`
} }
func (a AppConfig) getSchedule() (string, error) { func (a AppConfig) getSchedule() (schedule string, err error) {
var schedule string
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.getSchedule : %s : Start", a.Name) log.Printf("AppConfig.getSchedule : %s : Start", a.Name)
} }
@ -36,38 +35,42 @@ func (a AppConfig) getSchedule() (string, error) {
} }
} }
var ok bool
if *schedFlag != "" { if *schedFlag != "" {
schedule = *schedFlag schedule = *schedFlag
} else if a.needYearlySnapshot() { } else if ok, err = a.needYearlySnapshot(); ok && err == nil {
schedule = "yearly" schedule = "yearly"
} else if a.needMonthlySnapshot() { } else if ok, err = a.needMonthlySnapshot(); ok && err == nil {
schedule = "monthly" schedule = "monthly"
} else if a.needWeeklySnapshot() { } else if ok, err = a.needWeeklySnapshot(); ok && err == nil {
schedule = "weekly" schedule = "weekly"
} else if a.needDailySnapshot() { } else if ok, err = a.needDailySnapshot(); ok && err == nil {
schedule = "daily" schedule = "daily"
} else if a.needHourlySnapshot() { } else if ok, err = a.needHourlySnapshot(); ok && err == nil {
schedule = "hourly" schedule = "hourly"
} else { } else {
return schedule, nil return
} }
if ret, ok := cfg.Zfsnap[schedule]; !ok { if ret, ok := cfg.Zfsnap[schedule]; !ok {
return "", fmt.Errorf("no retention for %s", schedule) schedule = ""
err = fmt.Errorf("no retention for %s", schedule)
} else { } else {
re := regexp.MustCompile(`^([0-9]+[ymwdhMs]{1}|forever)$`) re := regexp.MustCompile(`^([0-9]+[ymwdhMs]{1}|forever)$`)
if !re.MatchString(ret) { if !re.MatchString(ret) {
return "", fmt.Errorf("wrong retention format for %s", schedule) schedule = ""
err = fmt.Errorf("wrong retention format for %s", schedule)
} }
} }
return schedule, nil return
} }
func (a AppConfig) needYearlySnapshot() bool { func (a AppConfig) needYearlySnapshot() (ret bool, err error) {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : Start", a.Name) log.Printf("AppConfig.needYearlySnapshot : %s : Start", a.Name)
} }
ret := false ret = false
// schedule enabled for app ? // schedule enabled for app ?
for _, v := range a.Schedule { for _, v := range a.Schedule {
@ -76,7 +79,7 @@ func (a AppConfig) needYearlySnapshot() bool {
} }
} }
if !ret { if !ret {
return false return
} }
// finding out the timestamps existing // finding out the timestamps existing
@ -85,15 +88,25 @@ func (a AppConfig) needYearlySnapshot() bool {
re := regexp.MustCompile(`^yearly-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) re := regexp.MustCompile(`^yearly-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources { for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{}) timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() { if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) { if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}") dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) log.Printf("AppConfig.needYearlySnapshot : %s : time.ParseInLocation(%s) : %s", a.Name, dateString, err)
} }
return
} else { } else {
timeSource[string(src)][dateTime] = struct{}{} timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{} timeTotal[dateTime] = struct{}{}
@ -115,20 +128,20 @@ func (a AppConfig) needYearlySnapshot() bool {
// finding an eligible timestamp // finding an eligible timestamp
for t, _ := range timeTotal { for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() { if t.Year() == cfg.Now.Year() {
return false ret = false
} }
} }
// no timestamp => need the snapshot ! // no timestamp => need the snapshot !
return true return
} }
func (a AppConfig) needMonthlySnapshot() bool { func (a AppConfig) needMonthlySnapshot() (ret bool, err error) {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needMonthlySnapshot : %s : Start", a.Name) log.Printf("AppConfig.needMonthlySnapshot : %s : Start", a.Name)
} }
ret := false ret = false
// schedule enabled for app ? // schedule enabled for app ?
for _, v := range a.Schedule { for _, v := range a.Schedule {
@ -137,7 +150,7 @@ func (a AppConfig) needMonthlySnapshot() bool {
} }
} }
if !ret { if !ret {
return false return
} }
// finding out the timestamps existing // finding out the timestamps existing
@ -146,15 +159,25 @@ func (a AppConfig) needMonthlySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) re := regexp.MustCompile(`^(yearly|monthly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources { for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{}) timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() { if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) { if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}") dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
} }
return
} else { } else {
timeSource[string(src)][dateTime] = struct{}{} timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{} timeTotal[dateTime] = struct{}{}
@ -176,20 +199,20 @@ func (a AppConfig) needMonthlySnapshot() bool {
// finding an eligible timestamp // finding an eligible timestamp
for t, _ := range timeTotal { for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() { if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() {
return false ret = false
} }
} }
// no timestamp => need the snapshot ! // no timestamp => need the snapshot !
return true return
} }
func (a AppConfig) needWeeklySnapshot() bool { func (a AppConfig) needWeeklySnapshot() (ret bool, err error) {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needWeeklySnapshot : %s : Start", a.Name) log.Printf("AppConfig.needWeeklySnapshot : %s : Start", a.Name)
} }
ret := false ret = false
// schedule enabled for app ? // schedule enabled for app ?
for _, v := range a.Schedule { for _, v := range a.Schedule {
@ -198,7 +221,7 @@ func (a AppConfig) needWeeklySnapshot() bool {
} }
} }
if !ret { if !ret {
return false return
} }
// finding out the timestamps existing // finding out the timestamps existing
@ -207,15 +230,25 @@ func (a AppConfig) needWeeklySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly|weekly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) re := regexp.MustCompile(`^(yearly|monthly|weekly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources { for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{}) timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() { if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) { if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}") dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
} }
return
} else { } else {
timeSource[string(src)][dateTime] = struct{}{} timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{} timeTotal[dateTime] = struct{}{}
@ -239,20 +272,20 @@ func (a AppConfig) needWeeklySnapshot() bool {
for t, _ := range timeTotal { for t, _ := range timeTotal {
snapYear, snapWeek := t.ISOWeek() snapYear, snapWeek := t.ISOWeek()
if nowYear == snapYear && nowWeek == snapWeek { if nowYear == snapYear && nowWeek == snapWeek {
return false ret = false
} }
} }
// no timestamp => need the snapshot ! // no timestamp => need the snapshot !
return true return
} }
func (a AppConfig) needDailySnapshot() bool { func (a AppConfig) needDailySnapshot() (ret bool, err error) {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needDailySnapshot : %s : Start", a.Name) log.Printf("AppConfig.needDailySnapshot : %s : Start", a.Name)
} }
ret := false ret = false
// schedule enabled for app ? // schedule enabled for app ?
for _, v := range a.Schedule { for _, v := range a.Schedule {
@ -261,7 +294,7 @@ func (a AppConfig) needDailySnapshot() bool {
} }
} }
if !ret { if !ret {
return false return
} }
// finding out the timestamps existing // finding out the timestamps existing
@ -270,15 +303,25 @@ func (a AppConfig) needDailySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly|weekly|daily)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) re := regexp.MustCompile(`^(yearly|monthly|weekly|daily)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources { for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{}) timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() { if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) { if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}") dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
} }
return
} else { } else {
timeSource[string(src)][dateTime] = struct{}{} timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{} timeTotal[dateTime] = struct{}{}
@ -300,20 +343,20 @@ func (a AppConfig) needDailySnapshot() bool {
// finding an eligible timestamp // finding an eligible timestamp
for t, _ := range timeTotal { for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() { if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() {
return false ret = false
} }
} }
// no timestamp => need the snapshot ! // no timestamp => need the snapshot !
return true return
} }
func (a AppConfig) needHourlySnapshot() bool { func (a AppConfig) needHourlySnapshot() (ret bool, err error) {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needHourlySnapshot : %s : Start", a.Name) log.Printf("AppConfig.needHourlySnapshot : %s : Start", a.Name)
} }
ret := false ret = false
// schedule enabled for app ? // schedule enabled for app ?
for _, v := range a.Schedule { for _, v := range a.Schedule {
@ -322,7 +365,7 @@ func (a AppConfig) needHourlySnapshot() bool {
} }
} }
if !ret { if !ret {
return false return
} }
// finding out the timestamps existing // finding out the timestamps existing
@ -331,15 +374,25 @@ func (a AppConfig) needHourlySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly|weekly|daily|hourly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`) re := regexp.MustCompile(`^(yearly|monthly|weekly|daily|hourly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources { for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{}) timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() { if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) { if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}") dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err) log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
} }
return
} else { } else {
timeSource[string(src)][dateTime] = struct{}{} timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{} timeTotal[dateTime] = struct{}{}
@ -361,13 +414,13 @@ func (a AppConfig) needHourlySnapshot() bool {
// finding an eligible timestamp // finding an eligible timestamp
for t, _ := range timeTotal { for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() && t.Hour() == cfg.Now.Hour() { if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() && t.Hour() == cfg.Now.Hour() {
return false ret = false
} }
} }
// no timestamp => need the snapshot ! // no timestamp => need the snapshot !
return true return
} }
func (a AppConfig) CheckZFS() error { func (a AppConfig) CheckZFS() error {
@ -480,7 +533,7 @@ func (a AppConfig) RefreshSnapshot() error {
return nil return nil
} }
func (a AppConfig) SendSnapshots() error { func (a AppConfig) SendSnapshots() (err error) {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Start", a.Name) log.Printf("AppConfig.SendSnapshots : %s : Start", a.Name)
} }
@ -490,18 +543,23 @@ func (a AppConfig) SendSnapshots() error {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending snapshots from %s to %s", a.Name, string(src), string(dest)) log.Printf("AppConfig.SendSnapshots : %s : Sending snapshots from %s to %s", a.Name, string(src), string(dest))
} }
dLastSnapshot, err := cfg.Box[dest.Box()].ZFSGetLastSnapshot(dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil { var dLastSnapshot Snapshot
dLastSnapshot, err = cfg.Box[dest.Box()].ZFSGetLastSnapshot(dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil && err.Error() == "no snapshot" {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s on %s", a.Name, string(src), dest.Box()) log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s on %s", a.Name, string(src), dest.Box())
} }
sFirstSnapshot, err := cfg.Box[src.Box()].ZFSGetFirstSnapshot(src.Path())
var sFirstSnapshot Snapshot
sFirstSnapshot, err = cfg.Box[src.Box()].ZFSGetFirstSnapshot(src.Path())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s", a.Name, string(src)) log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s", a.Name, string(src))
} }
return err return
} }
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s", a.Name, dest.Box(), string(sFirstSnapshot)) log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s", a.Name, dest.Box(), string(sFirstSnapshot))
} }
@ -510,23 +568,32 @@ func (a AppConfig) SendSnapshots() error {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sFirstSnapshot), err) log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sFirstSnapshot), err)
} }
return err return
} }
var sCurrSnapshot Snapshot
sNextSnapshot := sFirstSnapshot var (
for !cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) { sCurrSnapshot, sNextSnapshot Snapshot
isLastSnapshot bool
)
sNextSnapshot = sFirstSnapshot
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
for !isLastSnapshot {
sCurrSnapshot = sNextSnapshot sCurrSnapshot = sNextSnapshot
sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot) sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot)
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box()) log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box())
} }
if err != nil && err.Error() != "no snapshot" {
return
}
_, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path()) _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err) log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err)
} }
return err return
} }
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
} }
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : All snapshots sent for %s", a.Name, string(src)) log.Printf("AppConfig.SendSnapshots : %s : All snapshots sent for %s", a.Name, string(src))
@ -536,26 +603,35 @@ func (a AppConfig) SendSnapshots() error {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Last snapshot on %s is %s", a.Name, dest.Box(), string(dLastSnapshot)) log.Printf("AppConfig.SendSnapshots : %s : Last snapshot on %s is %s", a.Name, dest.Box(), string(dLastSnapshot))
} }
var sCurrSnapshot Snapshot var (
sNextSnapshot := Snapshot(string(dLastSnapshot)[len(dest.Path())+len(src.Box())+2:]) sCurrSnapshot, sNextSnapshot Snapshot
for !cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) { isLastSnapshot bool
)
sNextSnapshot = Snapshot(string(dLastSnapshot)[len(dest.Path())+len(src.Box())+2:])
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
for !isLastSnapshot {
sCurrSnapshot = sNextSnapshot sCurrSnapshot = sNextSnapshot
sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot) sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot)
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box()) log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box())
} }
if err != nil && err.Error() != "no snapshot" {
return
}
_, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path()) _, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil { if err != nil {
if *debugFlag { if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err) log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err)
} }
return err return
}
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
} }
} }
} }
} }
} err = nil
return nil return
} }
func (a AppConfig) CleanupSnapshot() error { func (a AppConfig) CleanupSnapshot() error {

157
box.go
View File

@ -17,28 +17,165 @@ type Box struct {
zfs *ZFSConfig zfs *ZFSConfig
} }
func (b *Box) ZFSGetLastSnapshot(path string) (s Snapshot, err error) { func (b *Box) ZFSGetLastSnapshot(path string) (last Snapshot, err error) {
return b.ssh.getLastSnapshot(path) err = b.SnapshotInitialize()
if err != nil {
return
}
if *debugFlag {
log.Printf("Box.ZFSGetLastSnapshot : %s : Start %s (%d snapshots)", b.Name, path, len(b.zfs.SnapshotList))
}
for _, v := range b.zfs.SnapshotList {
if v.Path() == path {
last = v
}
}
if len(string(last)) == 0 {
err = fmt.Errorf("no snapshot")
}
return
} }
func (b *Box) ZFSIsLastSnapshot(s Snapshot) bool { func (b *Box) ZFSIsLastSnapshot(src Snapshot) (is bool, err error) {
return b.ssh.isLastSnapshot(s) err = b.SnapshotInitialize()
if err != nil {
return
}
if *debugFlag {
log.Printf("SSHConfig.isLastSnapshot : %s : Start %s", b.Name, string(src))
}
_, err = b.ZFSGetNextSnapshot(src)
if err != nil {
if err.Error() == "no snapshot" {
is = true
err = nil
}
} else {
is = false
}
return
} }
func (b *Box) ZFSGetFirstSnapshot(path string) (s Snapshot, err error) { func (b *Box) ZFSGetFirstSnapshot(path string) (first Snapshot, err error) {
return b.ssh.getFirstSnapshot(path) err = b.SnapshotInitialize()
if err != nil {
return
}
if *debugFlag {
log.Printf("SSHConfig.getFirstSnapshot : Start %s:%s", b.Name, path)
}
for _, v := range b.zfs.SnapshotList {
if v.Path() == path {
first = v
return
}
}
err = fmt.Errorf("no snapshot")
return
} }
func (b *Box) ZFSGetNextSnapshot(src Snapshot) (next Snapshot, err error) { func (b *Box) ZFSGetNextSnapshot(src Snapshot) (next Snapshot, err error) {
return b.ssh.getNextSnapshot(src) err = b.SnapshotInitialize()
if err != nil {
return
}
if *debugFlag {
log.Printf("Box.ZFSGetNextSnapshot : Start %s:%s", b.Name, string(src))
}
for id, v := range b.zfs.SnapshotList {
if v == src {
if len(b.zfs.SnapshotList) > id+1 {
next = b.zfs.SnapshotList[id+1]
if next.Path() == src.Path() {
return
} else {
err = fmt.Errorf("no snapshot")
return
}
} else {
err = fmt.Errorf("no snapshot")
return
}
}
}
err = fmt.Errorf("no snapshot")
return
} }
func (b *Box) ZFSUpdateSnapshotList() (err error) { func (b *Box) ZFSUpdateSnapshotList() (err error) {
return b.ssh.getSnapshotList() b.zfs.M.Lock()
if b.zfs.SnapshotDeleted || b.zfs.SnapshotAdded {
b.zfs.SnapshotInitialized = false
}
b.zfs.M.Unlock()
err = b.SnapshotInitialize()
return
} }
func (b *Box) ZFSGetSnapshotList() []Snapshot { func (b *Box) ZFSGetSnapshotList() (snaps []Snapshot, err error) {
return b.ssh.snapshot err = b.SnapshotInitialize()
if err != nil {
return
}
b.zfs.M.Lock()
defer b.zfs.M.Unlock()
snaps = b.zfs.SnapshotList
return
}
func (b *Box) SnapshotInitialize() (err error) {
b.zfs.M.Lock()
defer b.zfs.M.Unlock()
if b.zfs.SnapshotInitialized {
return nil
}
if *debugFlag {
log.Printf("Box.SnapshotInitialize : %s : Start", b.Name)
}
b.zfs.SnapshotList = make([]Snapshot, 0)
var buf *bytes.Buffer
buf, err = b.SSHExec("zfs list -H -t snapshot -o name")
csvReader := csv.NewReader(buf)
csvReader.Comma = '\t'
csvReader.FieldsPerRecord = 1
csvData, err := csvReader.ReadAll()
if err != nil {
if *debugFlag {
log.Printf("Box.SnapshotInitialize : %s : csvReader.ReadAll() : %s", b.Name, err)
}
return err
}
for _, rec := range csvData {
b.zfs.SnapshotList = append(b.zfs.SnapshotList, Snapshot(rec[0]))
}
if *debugFlag {
log.Printf("Box.SnapshotInitialize : %s : read %d zfs snapshots", b.Name, len(b.zfs.SnapshotList))
}
b.zfs.SnapshotInitialized = true
b.zfs.SnapshotAdded = false
b.zfs.SnapshotDeleted = false
return nil
} }
func (b *Box) ZFSUpdateList() (err error) { func (b *Box) ZFSUpdateList() (err error) {

128
ssh.go
View File

@ -2,8 +2,6 @@ package main
import ( import (
"bytes" "bytes"
"encoding/csv"
"fmt"
"log" "log"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
@ -15,135 +13,9 @@ type SSHConfig struct {
client *ssh.Client client *ssh.Client
logged bool logged bool
name string name string
zfs map[string]string
snapshot []Snapshot snapshot []Snapshot
} }
func (s *SSHConfig) getLastSnapshot(path string) (Snapshot, error) {
if *debugFlag {
log.Printf("SSHConfig.getLastSnapshot : Start %s:%s (%d snapshots)", s.name, path, len(s.snapshot))
}
var last Snapshot
for _, v := range s.snapshot {
if v.Path() == path {
last = v
} else {
if len(string(last)) > 0 {
return last, nil
}
}
}
if len(string(last)) > 0 {
return last, nil
}
return last, fmt.Errorf("no snapshot")
}
func (s *SSHConfig) isLastSnapshot(snapshot Snapshot) bool {
if *debugFlag {
log.Printf("SSHConfig.isLastSnapshot : Start %s:%s", s.name, string(snapshot))
}
_, err := s.getNextSnapshot(snapshot)
if err != nil {
return true
} else {
return false
}
}
func (s *SSHConfig) getFirstSnapshot(path string) (Snapshot, error) {
if *debugFlag {
log.Printf("SSHConfig.getFirstSnapshot : Start %s:%s", s.name, path)
}
var first Snapshot
for _, v := range s.snapshot {
if v.Path() == path {
first = v
if *debugFlag {
log.Printf("SSHConfig.getFirstSnapshot : Return %s", string(first))
}
return first, nil
}
}
return first, fmt.Errorf("no snapshot")
}
func (s *SSHConfig) getNextSnapshot(snapshot Snapshot) (Snapshot, error) {
if *debugFlag {
log.Printf("SSHConfig.getNextSnapshot : Start %s:%s", s.name, string(snapshot))
}
var next Snapshot
for id, v := range s.snapshot {
if v == snapshot {
if len(s.snapshot) > id+1 {
next = s.snapshot[id+1]
if next.Path() == snapshot.Path() {
return next, nil
} else {
return next, fmt.Errorf("no snapshot")
}
} else {
return next, fmt.Errorf("no snapshot")
}
}
}
return next, fmt.Errorf("no snapshot")
}
func (s *SSHConfig) getSnapshotList() error {
if *debugFlag {
log.Printf("SSHConfig.getSnapshotList : %s : Start", s.name)
}
if !s.logged {
return fmt.Errorf("Client %s not logged in.", s.name)
}
session, err := s.client.NewSession()
if err != nil {
if *debugFlag {
log.Printf("SSHConfig.getSnapshotList : %s : client.NewSession() : %s", s.name, err)
}
return err
}
var b bytes.Buffer
session.Stdout = &b
err = session.Run("TZ=\"" + cfg.Timezone + "\" zfs list -H -t snapshot -o name")
if err != nil {
if *debugFlag {
log.Printf("SSHConfig.getSnapshotList : %s : session.Run() : %s", s.name, err)
}
return err
}
s.snapshot = make([]Snapshot, 0)
csvReader := csv.NewReader(&b)
csvReader.Comma = '\t'
csvReader.FieldsPerRecord = 1
csvData, err := csvReader.ReadAll()
if err != nil {
if *debugFlag {
log.Printf("SSHConfig.getSnapshotList : %s : csvReader.ReadAll() : %s", s.name, err)
}
return err
}
for _, rec := range csvData {
s.snapshot = append(s.snapshot, Snapshot(rec[0]))
}
if *debugFlag {
log.Printf("SSHConfig.getSnapshotList : %s : read %d zfs snapshots", s.name, len(s.snapshot))
}
session.Close()
return nil
}
func (s *SSHConfig) exec(cmd string) (b *bytes.Buffer, err error) { func (s *SSHConfig) exec(cmd string) (b *bytes.Buffer, err error) {
if *debugFlag { if *debugFlag {
log.Printf("SSHConfig.exec : %s : Start %s", s.name, cmd) log.Printf("SSHConfig.exec : %s : Start %s", s.name, cmd)

View File

@ -1,6 +1,6 @@
// Code generated by version.sh (@generated) DO NOT EDIT. // Code generated by version.sh (@generated) DO NOT EDIT.
package main package main
var githash = "2704149" var githash = "e192906"
var buildstamp = "2021-11-14_07:57:44" var buildstamp = "2021-11-14_09:20:16"
var commits = "20" var commits = "21"
var version = "2704149-b20 - 2021-11-14_07:57:44" var version = "e192906-b21 - 2021-11-14_09:20:16"