backup/app.go

package main

import (
	"fmt"
	"log"
	"regexp"
	"time"
)

type AppConfig struct {
	Name         string              `json:"name"`
	Schedule     []string            `json:"schedule"`
	Sources      []Location          `json:"src"`
	Destinations []Location          `json:"dest"`
	Before       map[string]Location `json:"before"`
	After        map[string]Location `json:"after"`
}

func (a AppConfig) getTime() time.Time {
	for _, v := range a.Sources {
		return cfg.Box[v.Box()].GetTime()
	}
	for _, v := range a.Destinations {
		return cfg.Box[v.Box()].GetTime()
	}
	return time.Now()
}

func (a AppConfig) getSchedule() (string, error) {
	var schedule string
	if *debugFlag {
		log.Printf("AppConfig.getSchedule : %s : Start", a.Name)
	}

	refreshSnapshot := make(map[string]bool)
	for _, v := range a.Sources {
		refreshSnapshot[v.Box()] = true
	}
	for k, _ := range refreshSnapshot {
		err := cfg.Box[k].ZFSUpdateSnapshotList()
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.getSchedule : %s : ZFSUpdateSnapshotList(%s) : %s", a.Name, k, err)
			}
			return "", err
		}
	}

	if *schedFlag != "" {
		schedule = *schedFlag
	} else if a.needYearlySnapshot() {
		schedule = "yearly"
	} else if a.needMonthlySnapshot() {
		schedule = "monthly"
	} else if a.needWeeklySnapshot() {
		schedule = "weekly"
	} else if a.needDailySnapshot() {
		schedule = "daily"
	} else if a.needHourlySnapshot() {
		schedule = "hourly"
	} else {
		return schedule, nil
	}

	if ret, ok := cfg.Zfsnap[schedule]; !ok {
		return "", fmt.Errorf("no retention for %s", schedule)
	} else {
		re := regexp.MustCompile(`^([0-9]+[ymwdhMs]{1}|forever)$`)
		if !re.MatchString(ret) {
			return "", fmt.Errorf("wrong retention format for %s", schedule)
		}
	}
	return schedule, nil
}

func (a AppConfig) needYearlySnapshot() bool {
	if *debugFlag {
		log.Printf("AppConfig.needYearlySnapshot : %s : Start", a.Name)
	}
	ret := false

	// schedule enabled for app ?
	for _, v := range a.Schedule {
		if v == "yearly" {
			ret = true
		}
	}
	if !ret {
		return false
	}

	// finding out the timestamps existing
	timeSource := make(map[string]map[time.Time]struct{})
	timeTotal := make(map[time.Time]struct{})
	re := regexp.MustCompile(`^yearly-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
	for _, src := range a.Sources {
		timeSource[string(src)] = make(map[time.Time]struct{})
		for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
			if src.Path() == snap.Path() {
				if re.MatchString(snap.Name()) {
					dateString := re.ReplaceAllString(snap.Name(), "${Date}")
					dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
					if err != nil {
						if *debugFlag {
							log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
						}
					} else {
						timeSource[string(src)][dateTime] = struct{}{}
						timeTotal[dateTime] = struct{}{}
					}
				}
			}
		}
	}

	// cleaning up the available timestamps for common timestamps
	for t, _ := range timeTotal {
		for _, v := range timeSource {
			if _, ok := v[t]; !ok {
				delete(timeTotal, t)
			}
		}
	}

	// finding an eligible timestamp
	now := a.getTime()
	for t, _ := range timeTotal {
		if t.Year() == now.Year() {
			return false
		}
	}

	// no timestamp => need the snapshot !

	return true
}

func (a AppConfig) needMonthlySnapshot() bool {
	if *debugFlag {
		log.Printf("AppConfig.needMonthlySnapshot : %s : Start", a.Name)
	}
	ret := false

	// schedule enabled for app ?
	for _, v := range a.Schedule {
		if v == "monthly" {
			ret = true
		}
	}
	if !ret {
		return false
	}

	// finding out the timestamps existing
	timeSource := make(map[string]map[time.Time]struct{})
	timeTotal := make(map[time.Time]struct{})
	re := regexp.MustCompile(`^(yearly|monthly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
	for _, src := range a.Sources {
		timeSource[string(src)] = make(map[time.Time]struct{})
		for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
			if src.Path() == snap.Path() {
				if re.MatchString(snap.Name()) {
					dateString := re.ReplaceAllString(snap.Name(), "${Date}")
					dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
					if err != nil {
						if *debugFlag {
							log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
						}
					} else {
						timeSource[string(src)][dateTime] = struct{}{}
						timeTotal[dateTime] = struct{}{}
					}
				}
			}
		}
	}

	// cleaning up the available timestamps for common timestamps
	for t, _ := range timeTotal {
		for _, v := range timeSource {
			if _, ok := v[t]; !ok {
				delete(timeTotal, t)
			}
		}
	}

	// finding an eligible timestamp
	now := a.getTime()
	for t, _ := range timeTotal {
		if t.Year() == now.Year() && t.Month() == now.Month() {
			return false
		}
	}

	// no timestamp => need the snapshot !

	return true
}

func (a AppConfig) needWeeklySnapshot() bool {
	if *debugFlag {
		log.Printf("AppConfig.needWeeklySnapshot : %s : Start", a.Name)
	}
	ret := false

	// schedule enabled for app ?
	for _, v := range a.Schedule {
		if v == "weekly" {
			ret = true
		}
	}
	if !ret {
		return false
	}

	// finding out the timestamps existing
	timeSource := make(map[string]map[time.Time]struct{})
	timeTotal := make(map[time.Time]struct{})
	re := regexp.MustCompile(`^(yearly|monthly|weekly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
	for _, src := range a.Sources {
		timeSource[string(src)] = make(map[time.Time]struct{})
		for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
			if src.Path() == snap.Path() {
				if re.MatchString(snap.Name()) {
					dateString := re.ReplaceAllString(snap.Name(), "${Date}")
					dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
					if err != nil {
						if *debugFlag {
							log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
						}
					} else {
						timeSource[string(src)][dateTime] = struct{}{}
						timeTotal[dateTime] = struct{}{}
					}
				}
			}
		}
	}

	// cleaning up the available timestamps for common timestamps
	for t, _ := range timeTotal {
		for _, v := range timeSource {
			if _, ok := v[t]; !ok {
				delete(timeTotal, t)
			}
		}
	}

	// finding an eligible timestamp
	now := a.getTime()
	nowYear, nowWeek := now.ISOWeek()
	for t, _ := range timeTotal {
		snapYear, snapWeek := t.ISOWeek()
		if nowYear == snapYear && nowWeek == snapWeek {
			return false
		}
	}

	// no timestamp => need the snapshot !

	return true
}

func (a AppConfig) needDailySnapshot() bool {
	if *debugFlag {
		log.Printf("AppConfig.needDailySnapshot : %s : Start", a.Name)
	}
	ret := false

	// schedule enabled for app ?
	for _, v := range a.Schedule {
		if v == "daily" {
			ret = true
		}
	}
	if !ret {
		return false
	}

	// finding out the timestamps existing
	timeSource := make(map[string]map[time.Time]struct{})
	timeTotal := make(map[time.Time]struct{})
	re := regexp.MustCompile(`^(yearly|monthly|weekly|daily)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
	for _, src := range a.Sources {
		timeSource[string(src)] = make(map[time.Time]struct{})
		for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
			if src.Path() == snap.Path() {
				if re.MatchString(snap.Name()) {
					dateString := re.ReplaceAllString(snap.Name(), "${Date}")
					dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
					if err != nil {
						if *debugFlag {
							log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
						}
					} else {
						timeSource[string(src)][dateTime] = struct{}{}
						timeTotal[dateTime] = struct{}{}
					}
				}
			}
		}
	}

	// cleaning up the available timestamps for common timestamps
	for t, _ := range timeTotal {
		for _, v := range timeSource {
			if _, ok := v[t]; !ok {
				delete(timeTotal, t)
			}
		}
	}

	// finding an eligible timestamp
	now := a.getTime()
	for t, _ := range timeTotal {
		if t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day() {
			return false
		}
	}

	// no timestamp => need the snapshot !

	return true
}

func (a AppConfig) needHourlySnapshot() bool {
	if *debugFlag {
		log.Printf("AppConfig.needHourlySnapshot : %s : Start", a.Name)
	}
	ret := false

	// schedule enabled for app ?
	for _, v := range a.Schedule {
		if v == "hourly" {
			ret = true
		}
	}
	if !ret {
		return false
	}

	// finding out the timestamps existing
	timeSource := make(map[string]map[time.Time]struct{})
	timeTotal := make(map[time.Time]struct{})
	re := regexp.MustCompile(`^(yearly|monthly|weekly|daily|hourly)-(?P<Date>[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}.[0-9]{2}.[0-9]{2})--([0-9]+[ymwdhMs]{1}|forever)$`)
	for _, src := range a.Sources {
		timeSource[string(src)] = make(map[time.Time]struct{})
		for _, snap := range cfg.Box[src.Box()].ZFSGetSnapshotList() {
			if src.Path() == snap.Path() {
				if re.MatchString(snap.Name()) {
					dateString := re.ReplaceAllString(snap.Name(), "${Date}")
					dateTime, err := time.Parse("2006-01-02_15.04.05", dateString)
					if err != nil {
						if *debugFlag {
							log.Printf("AppConfig.needYearlySnapshot : %s : time.Parse(%s) : %s", a.Name, dateString, err)
						}
					} else {
						timeSource[string(src)][dateTime] = struct{}{}
						timeTotal[dateTime] = struct{}{}
					}
				}
			}
		}
	}

	// cleaning up the available timestamps for common timestamps
	for t, _ := range timeTotal {
		for _, v := range timeSource {
			if _, ok := v[t]; !ok {
				delete(timeTotal, t)
			}
		}
	}

	// finding an eligible timestamp
	now := a.getTime()
	for t, _ := range timeTotal {
		if t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day() && t.Hour() == now.Hour() {
			return false
		}
	}

	// no timestamp => need the snapshot !

	return true
}

func (a AppConfig) CheckZFS() error {
	if *debugFlag {
		log.Printf("AppConfig.CheckZFS : %s : Start", a.Name)
	}

	for _, src := range a.Sources {
		if !cfg.Box[src.Box()].ZFSIsZFS(src.Path()) {
			return fmt.Errorf("No path %s on source", string(src))
		}
		for _, dest := range a.Destinations {
			if !cfg.Box[dest.Box()].ZFSIsZFS(dest.Path() + "/" + src.Box() + "/" + src.Path()) {
				err := cfg.Box[dest.Box()].ZFSCreateZFS(dest.Path() + "/" + src.Box() + "/" + src.Path())
				if err != nil {
					if *debugFlag {
						log.Printf("AppConfig.CheckZFS : %s : Error creating %s on %s", a.Name, dest.Path()+"/"+src.Box()+"/"+src.Path(), dest.Box())
					}
					return err
				}
			}
		}
	}
	return nil
}

func (a AppConfig) ExecBefore(schedule string) error {
	if *debugFlag {
		log.Printf("AppConfig.ExecBefore : %s : Start %s", a.Name, schedule)
	}

	for k, v := range a.Before {
		re := regexp.MustCompile(k)
		if re.MatchString(schedule) {
			err := cfg.Box[v.Box()].SSHExec(v.Path())
			if err != nil {
				if *debugFlag {
					log.Printf("AppConfig.ExecBefore : %s : Error executing %s", a.Name, string(v))
				}
				return err
			}
		}
	}
	return nil
}

func (a AppConfig) ExecAfter(schedule string) error {
	if *debugFlag {
		log.Printf("AppConfig.ExecAfter : %s : Start %s", a.Name, schedule)
	}

	for k, v := range a.After {
		re := regexp.MustCompile(k)
		if re.MatchString(schedule) {
			err := cfg.Box[v.Box()].SSHExec(v.Path())
			if err != nil {
				if *debugFlag {
					log.Printf("AppConfig.ExecAfter : %s : Error executing %s on %s", a.Name, v.Path(), v.Box())
				}
				return err
			}
		}
	}

	return nil
}

func (a AppConfig) TakeSnapshot(schedule string) error {
	if *debugFlag {
		log.Printf("AppConfig.TakeSnapshot : %s : Start %s", a.Name, schedule)
	}

	takeSnapshot := make(map[string]string)
	for _, v := range a.Sources {
		takeSnapshot[v.Box()] = takeSnapshot[v.Box()] + " " + v.Path()
	}
	for k, v := range takeSnapshot {
		if *debugFlag {
			log.Printf("AppConfig.TakeSnapshot : %s : taking snapshot on %s for %s", a.Name, k, v)
		}
		err := cfg.Box[k].SSHExec("zfsnap snapshot -p " + schedule + "- -a " + cfg.Zfsnap[schedule] + v)
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.TakeSnapshot : %s : Error executing zfsnap on %s", a.Name, k)
			}
			return err
		}
	}

	return nil
}

func (a AppConfig) RefreshSnapshot() error {
	if *debugFlag {
		log.Printf("AppConfig.RefreshSnapshot : %s : Start", a.Name)
	}

	refreshSnapshot := make(map[string]struct{})
	for _, v := range a.Sources {
		refreshSnapshot[v.Box()] = struct{}{}
	}
	for _, v := range a.Destinations {
		refreshSnapshot[v.Box()] = struct{}{}
	}
	for k, _ := range refreshSnapshot {
		if *debugFlag {
			log.Printf("AppConfig.RefreshSnapshot : %s : refreshing snapshots for source %s", a.Name, k)
		}
		err := cfg.Box[k].ZFSUpdateSnapshotList()
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.RefreshSnapshot : %s : Error getting snapshots on %s", a.Name, k)
			}
			return err
		}
	}
	return nil
}

func (a AppConfig) SendSnapshots() error {
	if *debugFlag {
		log.Printf("AppConfig.SendSnapshots : %s : Start", a.Name)
	}

	for _, src := range a.Sources {
		for _, dest := range a.Destinations {
			if *debugFlag {
				log.Printf("AppConfig.SendSnapshots : %s : Sending snapshots from %s to %s", a.Name, string(src), string(dest))
			}
			dLastSnapshot, err := cfg.Box[dest.Box()].ZFSGetLastSnapshot(dest.Path() + "/" + src.Box() + "/" + src.Path())
			if err != nil {
				if *debugFlag {
					log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s on %s", a.Name, string(src), dest.Box())
				}
				sFirstSnapshot, err := cfg.Box[src.Box()].ZFSGetFirstSnapshot(src.Path())
				if err != nil {
					if *debugFlag {
						log.Printf("AppConfig.SendSnapshots : %s : No snapshot for %s", a.Name, string(src))
					}
					return err
				}
				if *debugFlag {
					log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s", a.Name, dest.Box(), string(sFirstSnapshot))
				}
				err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send " + string(sFirstSnapshot) + " | zfs recv -F " + dest.Path() + "/" + src.Box() + "/" + src.Path())
				if err != nil {
					if *debugFlag {
						log.Printf("AppConfig.SendSnapshots : %s : Initializing snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sFirstSnapshot), err)
					}
					return err
				}
				var sCurrSnapshot Snapshot
				sNextSnapshot := sFirstSnapshot
				for !cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) {
					sCurrSnapshot = sNextSnapshot
					sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot)
					if *debugFlag {
						log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box())
					}
					err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path())
					if err != nil {
						if *debugFlag {
							log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err)
						}
						return err
					}
				}
				if *debugFlag {
					log.Printf("AppConfig.SendSnapshots : %s : All snapshots sent for %s", a.Name, string(src))
				}

			} else {
				if *debugFlag {
					log.Printf("AppConfig.SendSnapshots : %s : Last snapshot on %s is %s", a.Name, dest.Box(), string(dLastSnapshot))
				}
				var sCurrSnapshot Snapshot
				sNextSnapshot := Snapshot(string(dLastSnapshot)[len(dest.Path())+len(src.Box())+2:])
				for !cfg.Box[src.Box()].ZFSIsLastSnapshot(sNextSnapshot) {
					sCurrSnapshot = sNextSnapshot
					sNextSnapshot, err = cfg.Box[src.Box()].ZFSGetNextSnapshot(sNextSnapshot)
					if *debugFlag {
						log.Printf("AppConfig.SendSnapshots : %s : Sending incrementally %s to %s", a.Name, string(sNextSnapshot), dest.Box())
					}
					err = cfg.Box[dest.Box()].SSHExec("ssh " + cfg.Box[src.Box()].User + "@" + src.Box() + " zfs send -I " + string(sCurrSnapshot) + " " + string(sNextSnapshot) + " | zfs recv " + dest.Path() + "/" + src.Box() + "/" + src.Path())
					if err != nil {
						if *debugFlag {
							log.Printf("AppConfig.SendSnapshots : %s : Sending snapshot on %s from %s failed (%s)", a.Name, dest.Box(), string(sNextSnapshot), err)
						}
						return err
					}
				}
			}
		}
	}
	return nil
}

func (a AppConfig) CleanupSnapshot() error {
	if *debugFlag {
		log.Printf("AppConfig.CleanupSnapshot : %s : Start", a.Name)
	}

	cleanupSnapshot := make(map[string]string)

	for _, dest := range a.Destinations {
		for _, src := range a.Sources {
			cleanupSnapshot[src.Box()] = cleanupSnapshot[src.Box()] + " " + src.Path()
			cleanupSnapshot[dest.Box()] = cleanupSnapshot[dest.Box()] + " " + dest.Path() + "/" + src.Box() + "/" + src.Path()
		}
	}

	for k, v := range cleanupSnapshot {
		if *debugFlag {
			log.Printf("AppConfig.CleanupSnapshot : %s : cleaning snapshots on %s for%s", a.Name, k, v)
		}
		err := cfg.Box[k].SSHExec("zfsnap destroy -p hourly- -p daily- -p weekly- -p monthly- -p yearly-" + v)
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.CleanupSnapshot : %s : Error executing zfsnap on %s", a.Name, k)
			}
			return err
		}
	}

	return nil
}

func (a AppConfig) RunAppBackup() error {
	if *debugFlag {
		log.Printf("AppConfig.RunAppBackup : %s : Start", a.Name)
	}
	schedule, err := a.getSchedule()
	if err != nil {
		if *debugFlag {
			log.Printf("AppConfig.RunAppBackup : %s : Error getting schedule : %s", a.Name, err)
		}
		return err
	}

	if schedule != "" || *slowFlag {
		err = a.CheckZFS()
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.RunAppBackup : %s : CheckZFS : %s", a.Name, err)
			}
			return err
		}
	}

	if schedule != "" {
		err = a.ExecBefore(schedule)
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.RunAppBackup : %s : ExecBefore : %s", a.Name, err)
			}
			return err
		}

		err = a.TakeSnapshot(schedule)
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.RunAppBackup : %s : TakeSnapshot : %s", a.Name, err)
			}
			return err
		}
	}

	err = a.RefreshSnapshot()
	if err != nil {
		if *debugFlag {
			log.Printf("AppConfig.RunAppBackup : %s : RefreshSnapshot : %s", a.Name, err)
		}
		return err
	}

	err = a.SendSnapshots()
	if err != nil {
		if *debugFlag {
			log.Printf("AppConfig.RunAppBackup : %s : SendSnapshots : %s", a.Name, err)
		}
		return err
	}

	if schedule != "" {
		err = a.ExecAfter(schedule)
		if err != nil {
			if *debugFlag {
				log.Printf("AppConfig.RunAppBackup : %s : ExecAfter : %s", a.Name, err)
			}
			return err
		}
	}

	err = a.CleanupSnapshot()
	if err != nil {
		if *debugFlag {
			log.Printf("AppConfig.RunAppBackup : %s : RefreshSnapshot : %s", a.Name, err)
		}
		return err
	}

	return nil
}