shoopea/backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update snapshot processing

Browse Source
This commit is contained in:
shoopea
2021-11-14 17:20:44 +08:00
parent e192906265
commit 0d3981ef4a
4 changed files with 289 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
app.go
View File

@@ -16,8 +16,7 @@ type AppConfig struct {
After map[string]Location `json:"after"`
}
func (a AppConfig) getSchedule() (string, error) {
var schedule string
func (a AppConfig) getSchedule() (schedule string, err error) {
if *debugFlag {
log.Printf("AppConfig.getSchedule : %s : Start", a.Name)
}
@@ -36,38 +35,42 @@ func (a AppConfig) getSchedule() (string, error) {
}
}
var ok bool
if *schedFlag != "" {
schedule = *schedFlag
} else if a.needYearlySnapshot() {
} else if ok, err = a.needYearlySnapshot(); ok && err == nil {
schedule = "yearly"
} else if a.needMonthlySnapshot() {
} else if ok, err = a.needMonthlySnapshot(); ok && err == nil {
schedule = "monthly"
} else if a.needWeeklySnapshot() {
} else if ok, err = a.needWeeklySnapshot(); ok && err == nil {
schedule = "weekly"
} else if a.needDailySnapshot() {
} else if ok, err = a.needDailySnapshot(); ok && err == nil {
schedule = "daily"
} else if a.needHourlySnapshot() {
} else if ok, err = a.needHourlySnapshot(); ok && err == nil {
schedule = "hourly"
} else {
return schedule, nil
return
}
if ret, ok := cfg.Zfsnap[schedule]; !ok {
return "", fmt.Errorf("no retention for %s", schedule)
schedule = ""
err = fmt.Errorf("no retention for %s", schedule)
} else {
re := regexp.MustCompile(`^([0-9]+[ymwdhMs]{1}|forever)$`)
if !re.MatchString(ret) {
return "", fmt.Errorf("wrong retention format for %s", schedule)
schedule = ""
err = fmt.Errorf("wrong retention format for %s", schedule)
}
}
return schedule, nil
return
}
func (a AppConfig) needYearlySnapshot() bool {
func (a AppConfig) needYearlySnapshot() (ret bool, err error) {
if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : Start", a.Name)
}
ret := false
ret = false
// schedule enabled for app ?
for _, v := range a.Schedule {
@@ -76,7 +79,7 @@ func (a AppConfig) needYearlySnapshot() bool {
}
}
if !ret {
return false
return
}
// finding out the timestamps existing
@@ -85,15 +88,25 @@ func (a AppConfig) needYearlySnapshot() bool {
re := regexp.MustCompile(`^yearly-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
log.Printf("AppConfig.needYearlySnapshot : %s : time.ParseInLocation(%s) : %s", a.Name, dateString, err)
}
return
} else {
timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{}
@@ -115,20 +128,20 @@ func (a AppConfig) needYearlySnapshot() bool {
// finding an eligible timestamp
for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() {
return false
ret = false
}
}
// no timestamp => need the snapshot !
return true
return
}
func (a AppConfig) needMonthlySnapshot() bool {
func (a AppConfig) needMonthlySnapshot() (ret bool, err error) {
if *debugFlag {
log.Printf("AppConfig.needMonthlySnapshot : %s : Start", a.Name)
}
ret := false
ret = false
// schedule enabled for app ?
for _, v := range a.Schedule {
@@ -137,7 +150,7 @@ func (a AppConfig) needMonthlySnapshot() bool {
}
}
if !ret {
return false
return
}
// finding out the timestamps existing
@@ -146,15 +159,25 @@ func (a AppConfig) needMonthlySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
}
return
} else {
timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{}
@@ -176,20 +199,20 @@ func (a AppConfig) needMonthlySnapshot() bool {
// finding an eligible timestamp
for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() {
return false
ret = false
}
}
// no timestamp => need the snapshot !
return true
return
}
func (a AppConfig) needWeeklySnapshot() bool {
func (a AppConfig) needWeeklySnapshot() (ret bool, err error) {
if *debugFlag {
log.Printf("AppConfig.needWeeklySnapshot : %s : Start", a.Name)
}
ret := false
ret = false
// schedule enabled for app ?
for _, v := range a.Schedule {
@@ -198,7 +221,7 @@ func (a AppConfig) needWeeklySnapshot() bool {
}
}
if !ret {
return false
return
}
// finding out the timestamps existing
@@ -207,15 +230,25 @@ func (a AppConfig) needWeeklySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly|weekly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
}
return
} else {
timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{}
@@ -239,20 +272,20 @@ func (a AppConfig) needWeeklySnapshot() bool {
for t, _ := range timeTotal {
snapYear, snapWeek := t.ISOWeek()
if nowYear == snapYear && nowWeek == snapWeek {
return false
ret = false
}
}
// no timestamp => need the snapshot !
return true
return
}
func (a AppConfig) needDailySnapshot() bool {
func (a AppConfig) needDailySnapshot() (ret bool, err error) {
if *debugFlag {
log.Printf("AppConfig.needDailySnapshot : %s : Start", a.Name)
}
ret := false
ret = false
// schedule enabled for app ?
for _, v := range a.Schedule {
@@ -261,7 +294,7 @@ func (a AppConfig) needDailySnapshot() bool {
}
}
if !ret {
return false
return
}
// finding out the timestamps existing
@@ -270,15 +303,25 @@ func (a AppConfig) needDailySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly|weekly|daily)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
}
return
} else {
timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{}
@@ -300,20 +343,20 @@ func (a AppConfig) needDailySnapshot() bool {
// finding an eligible timestamp
for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() {
return false
ret = false
}
}
// no timestamp => need the snapshot !
return true
return
}
func (a AppConfig) needHourlySnapshot() bool {
func (a AppConfig) needHourlySnapshot() (ret bool, err error) {
if *debugFlag {
log.Printf("AppConfig.needHourlySnapshot : %s : Start", a.Name)
}
ret := false
ret = false
// schedule enabled for app ?
for _, v := range a.Schedule {
@@ -322,7 +365,7 @@ func (a AppConfig) needHourlySnapshot() bool {
}
}
if !ret {
return false
return
}
// finding out the timestamps existing
@@ -331,15 +374,25 @@ func (a AppConfig) needHourlySnapshot() bool {
re := regexp.MustCompile(`^(yearly|monthly|weekly|daily|hourly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
for _, src := range a.Sources {
timeSource[string(src)] = make(map[time.Time]struct{})
for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
var snapList []Snapshot
snapList, err = cfg.Box[src.Box()].ZFSGetSnapshotList()
if err != nil {
return
}
for _, snap := range snapList {
if src.Path() == snap.Path() {
if re.MatchString(snap.Name()) {
dateString := re.ReplaceAllString(snap.Name(), "${Date}")
dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
var dateTime time.Time
dateTime, err = time.ParseInLocation("2006-01-02_15.04.05", dateString, time.Now().Location())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
}
return
} else {
timeSource[string(src)][dateTime] = struct{}{}
timeTotal[dateTime] = struct{}{}
@@ -361,13 +414,13 @@ func (a AppConfig) needHourlySnapshot() bool {
// finding an eligible timestamp
for t, _ := range timeTotal {
if t.Year() == cfg.Now.Year() && t.Month() == cfg.Now.Month() && t.Day() == cfg.Now.Day() && t.Hour() == cfg.Now.Hour() {
return false
ret = false
}
}
// no timestamp => need the snapshot !
return true
return
}
func (a AppConfig) CheckZFS() error {
@@ -480,7 +533,7 @@ func (a AppConfig) RefreshSnapshot() error {
return nil
}
func (a AppConfig) SendSnapshots() error {
func (a AppConfig) SendSnapshots() (err error) {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Start", a.Name)
}
@@ -490,18 +543,23 @@ func (a AppConfig) SendSnapshots() error {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending snapshots from %s to %s", a.Name, string(src), string(dest))
}
dLastSnapshot, err := cfg.Box[dest.Box()].ZFSGetLastSnapshot(dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil {
var dLastSnapshot Snapshot
dLastSnapshot, err = cfg.Box[dest.Box()].ZFSGetLastSnapshot(dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil && err.Error() == "no snapshot" {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s on %s", a.Name, string(src), dest.Box())
}
sFirstSnapshot, err := cfg.Box[src.Box()].ZFSGetFirstSnapshot(src.Path())
var sFirstSnapshot Snapshot
sFirstSnapshot, err = cfg.Box[src.Box()].ZFSGetFirstSnapshot(src.Path())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s", a.Name, string(src))
}
return err
return
}
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s", a.Name, dest.Box(), string(sFirstSnapshot))
}
@@ -510,23 +568,32 @@ func (a AppConfig) SendSnapshots() error {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sFirstSnapshot), err)
}
return err
return
}
var sCurrSnapshot Snapshot
sNextSnapshot := sFirstSnapshot
for !cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) {
var (
sCurrSnapshot, sNextSnapshot Snapshot
isLastSnapshot bool
)
sNextSnapshot = sFirstSnapshot
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
for !isLastSnapshot {
sCurrSnapshot = sNextSnapshot
sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot)
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box())
}
if err != nil && err.Error() != "no snapshot" {
return
}
_, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err)
}
return err
return
}
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
}
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : All snapshots sent for %s", a.Name, string(src))
@@ -536,26 +603,35 @@ func (a AppConfig) SendSnapshots() error {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Last snapshot on %s is %s", a.Name, dest.Box(), string(dLastSnapshot))
}
var sCurrSnapshot Snapshot
sNextSnapshot := Snapshot(string(dLastSnapshot)[len(dest.Path())+len(src.Box())+2:])
for !cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) {
var (
sCurrSnapshot, sNextSnapshot Snapshot
isLastSnapshot bool
)
sNextSnapshot = Snapshot(string(dLastSnapshot)[len(dest.Path())+len(src.Box())+2:])
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
for !isLastSnapshot {
sCurrSnapshot = sNextSnapshot
sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot)
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box())
}
if err != nil && err.Error() != "no snapshot" {
return
}
_, err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path())
if err != nil {
if *debugFlag {
log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err)
}
return err
return
}
isLastSnapshot, _ = cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot)
}
}
}
}
return nil
err = nil
return
}
func (a AppConfig) CleanupSnapshot() error {