1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#!/usr/bin/env python
# Copyright (C) 2006-2007 Richard Purdie
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License version 2 as published by the Free
# Software Foundation;
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
import optparse
import os, sys, stat
__version__ = "0.0.1"
def write_cache(cachefile, cachedata):
fd = open(cachefile, 'w')
for f in cachedata:
s = f + '|' + str(cachedata[f]['ts']) + '|' + str(cachedata[f]['size'])
fd.write(s + '\n')
fd.close()
def read_cache(cachefile):
cache = {}
f = open(cachefile, 'r')
lines = f.readlines()
f.close()
for l in lines:
data = l.split('|')
cache[data[0]] = {}
cache[data[0]]['ts'] = int(data[1])
cache[data[0]]['size'] = int(data[2])
cache[data[0]]['seen'] = False
return cache
def mkdirhier(dir):
"""Create a directory like 'mkdir -p', but does not complain if
directory already exists like os.makedirs
"""
try:
os.makedirs(dir)
except OSError, e:
if e.errno != 17: raise e
if __name__ == "__main__":
parser = optparse.OptionParser( version = "Metadata Stage Manager version %s" % ( __version__ ),
usage = """%prog [options]\n\nPerforms management tasks on a metadata staging area.""" )
parser.add_option( "-p", "--parentdir", help = "the path to the metadata parent directory",
action = "store", dest = "parentdir", default = None)
parser.add_option( "-c", "--cachefile", help = "the cache file to use",
action = "store", dest = "cachefile", default = None)
parser.add_option( "-d", "--copydir", help = "copy changed files to this directory",
action = "store", dest = "copydir", default = None)
parser.add_option( "-u", "--update", help = "update the cache file",
action = "store_true", dest = "update", default = False)
(options, args) = parser.parse_args()
if options.parentdir is None:
print("Error, --parentdir option not supplied")
sys.exit(1)
if options.cachefile is None:
print("Error, --cachefile option not supplied")
sys.exit(1)
if not options.parentdir.endswith('/'):
options.parentdir = options.parentdir + '/'
cache = {}
if os.access(options.cachefile, os.F_OK):
cache = read_cache(options.cachefile)
found_difference = False
def updateCache(path, fstamp):
cache[path] = {}
cache[path]['ts'] = fstamp[stat.ST_MTIME]
cache[path]['size'] = fstamp[stat.ST_SIZE]
cache[path]['seen'] = True
found_difference = True
def copyfile(path):
if options.copydir:
copypath = os.path.join(options.copydir, path.replace(options.parentdir, '', 1))
mkdirhier(os.path.split(copypath)[0])
os.system("cp -Pp " + path + " " + copypath)
def copydir(path, fstamp):
if options.copydir:
copypath = os.path.join(options.copydir, path.replace(options.parentdir, '', 1))
if os.path.exists(copypath):
os.system("rm -rf " + copypath)
if os.path.islink(path):
os.symlink(os.readlink(path), copypath)
else:
mkdirhier(copypath)
os.utime(copypath, (fstamp[stat.ST_ATIME], fstamp[stat.ST_MTIME]))
for root, dirs, files in os.walk(options.parentdir):
for f in files:
path = os.path.join(root, f)
if not os.access(path, os.R_OK):
continue
fstamp = os.lstat(path)
if path not in cache:
print "new file %s" % path
updateCache(path, fstamp)
copyfile(path)
else:
if cache[path]['ts'] != fstamp[stat.ST_MTIME] or cache[path]['size'] != fstamp[stat.ST_SIZE]:
print "file %s changed" % path
updateCache(path, fstamp)
copyfile(path)
cache[path]['seen'] = True
for d in dirs:
path = os.path.join(root, d)
fstamp = os.lstat(path)
if path not in cache:
print "new dir %s" % path
updateCache(path, fstamp)
copydir(path, fstamp)
else:
if cache[path]['ts'] != fstamp[stat.ST_MTIME]:
print "dir %s changed" % path
updateCache(path, fstamp)
copydir(path, fstamp)
cache[path]['seen'] = True
todel = []
for path in cache:
if not cache[path]['seen']:
print "%s removed" % path
found_difference = True
todel.append(path)
if options.update:
print "Updating"
for path in todel:
del cache[path]
mkdirhier(os.path.split(options.cachefile)[0])
write_cache(options.cachefile, cache)
if found_difference:
sys.exit(5)
sys.exit(0)
|
