remux.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. #!/usr/bin/env python3
  2. import sys, re, os, subprocess, pathlib, xattr, langcodes, logging
  3. import probe
  4. # Static config
  5. debug = False
  6. mux_xattr = 'zib.mux'
  7. mux_vers = 1
  8. lang_xattr = 'zib.lang'
  9. lang_vers = 1
  10. min_size = 0.9
  11. lang_default = 'eng'
  12. propedit_exec = 'mkvpropedit'
  13. re_subs = re.compile(r'(?:\.(\w\w))?(?:\.(sdh))?\.srt$', re.IGNORECASE)
  14. re_media = re.compile(r'^(.*?)(?:\.\w\w)?(?:\.sdh)?\.((?:srt)|(?:mp4|m4v|mkv))$', re.IGNORECASE)
  15. # ENV config
  16. if 'DEBUG' in os.environ:
  17. debug = bool(os.environ['DEBUG'])
  18. if 'PROPEDIT' in os.environ:
  19. propedit_exec = Pathlib.path(os.environ['PROPEDIT'])
  20. # Logging
  21. logger = logging.getLogger(__name__)
  22. if debug:
  23. logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
  24. else:
  25. logging.basicConfig(level=logging.WARN, stream=sys.stderr)
  26. # CLI config
  27. if len(sys.argv) < 2:
  28. print(f"Usage: {sys.argv[0]} base_directory", file=sys.stderr)
  29. sys.exit(-1)
  30. root = pathlib.Path(sys.argv[1])
  31. logger.debug(f"Root: {root}")
  32. def read_xattr(path, attr):
  33. value = None
  34. try:
  35. if attr in xattr.listxattr(path):
  36. value = int.from_bytes(xattr.getxattr(path, attr))
  37. logger.debug(f"got xattr: {attr} => {value}")
  38. except BaseException as e:
  39. logger.error(f"xattr error ({attr}): {e}")
  40. return value
  41. def write_xattr(path, attr, value):
  42. error = True
  43. try:
  44. xattr.setxattr(path, attr, value.to_bytes())
  45. error = False
  46. logger.debug(f"set xattr: {attr} => {value}")
  47. except BaseException as e:
  48. logger.error(f"xattr error ({attr}): {e}")
  49. return not error
  50. def scan(files):
  51. logger.debug(f"Scanning {len(files)} files")
  52. media = []
  53. # Scan this directory for media
  54. for file in files:
  55. info = {
  56. 'name': file,
  57. 'path': parent.joinpath(file),
  58. }
  59. # Parse media paths
  60. try:
  61. m = re_media.search(info['name'])
  62. if m:
  63. info['base'] = m.group(1)
  64. info['ext'] = m.group(2).lower()
  65. info['g_base'] = info['base'].lower()
  66. except BaseException as e:
  67. logger.error(f"Skipping file due to name error ({info['name']}): {e}")
  68. continue
  69. # Skip non-media files
  70. if 'ext' not in info:
  71. logger.debug(f"Skipping non-media file: {info['name']}")
  72. continue
  73. # Read xattrs
  74. info['mux_vers'] = read_xattr(info['path'], mux_xattr)
  75. info['lang_vers'] = read_xattr(info['path'], lang_xattr)
  76. logger.debug(f"Found media: {info}")
  77. media.append(info)
  78. # Return
  79. return media
  80. def mux(media):
  81. logger.debug(f"Muxing {len(media)} files")
  82. # Find groups
  83. groups = {}
  84. for file in media:
  85. if 'g_base' in file:
  86. g_base = file['g_base']
  87. if file['g_base'] not in groups:
  88. groups[g_base] = []
  89. if file not in groups[g_base]:
  90. groups[g_base].append(file)
  91. # Merge groups
  92. for (g_base, group) in groups.items():
  93. logger.debug(f"Merging group: {group[0]['base']}")
  94. # Skip groups that do not need muxing
  95. if len(group) == 1:
  96. if 'mux_vers' in group[0] and group[0]['mux_vers'] and group[0]['mux_vers'] >= mux_vers:
  97. logger.debug(f"Skipping single-item group with current mux_vers: {group[0]['path']}")
  98. continue
  99. # Construct args and file lists
  100. size = 0
  101. sources = []
  102. subs = []
  103. videos = []
  104. temp = group[0]['path'].parent.joinpath(f'{group[0]['base']}-remux.mkv')
  105. final = group[0]['path'].parent.joinpath(f'{group[0]['base']}.mkv')
  106. args = [ 'mkvmerge', '--default-language', lang_default, '-o', temp ]
  107. for file in group:
  108. # Track all source files
  109. size += file['path'].stat().st_size
  110. sources.append(file)
  111. # Find the video source
  112. if file['ext'] in [ 'mkv', 'mp4', 'm4v' ]:
  113. logger.debug(f"Found video: {file['path'].name}")
  114. videos.append(file)
  115. # Parse subtitle languages
  116. else:
  117. logger.debug(f"Found subtitle: {file['path'].name}")
  118. sub = { 'path': file['path'], 'lang': lang_default, 'sdh': False }
  119. m = re_subs.search(file['path'].name)
  120. if m:
  121. for part in m.groups():
  122. if part.lower() == 'sdh':
  123. sub['sdh'] = True
  124. else:
  125. code = langcodes.Language.get(part.lower())
  126. if code:
  127. sub['lang'] = code.to_alpha3()
  128. else:
  129. logger.error(f"Cound not match language ({file['path']}): {part}")
  130. logger.debug(f"Using language: {sub['lang']}")
  131. subs.append(sub)
  132. # We should find exactly one video source
  133. if len(videos) != 1:
  134. logger.error(f"Did not find exactly 1 video: {group[0]['base']}")
  135. continue
  136. video = videos[0]
  137. del videos
  138. # Video input file
  139. args.append(video['path'])
  140. # Subtitles get language and SDH metadata
  141. # Index is always 0; these files contain only one stream
  142. for sub in subs:
  143. args.extend([ '--language', f"0:{sub['lang']}" ])
  144. if sub['sdh']:
  145. args.extend([ '--hearing-impaired-flag', '0' ])
  146. args.append(sub['path'])
  147. del subs
  148. # mkvmerge
  149. if temp.exists():
  150. logger.error(f"Skipping because mux output exists: {temp}")
  151. continue
  152. logger.debug(f"Mux command: {args}")
  153. merge = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  154. logger.debug(f"Mux result: {merge}")
  155. # Verify
  156. if merge.returncode != 0 or not temp.exists() or temp.stat().st_size < (min_size * size):
  157. logger.error(f"Mux error: {merge.stdout.decode('utf-8')}")
  158. continue
  159. # Set xattr
  160. if not write_xattr(temp, mux_xattr, mux_vers):
  161. continue
  162. video['mux_vers'] = mux_vers
  163. # Remove sources
  164. try:
  165. for source in sources:
  166. logger.debug(f"Removing: {source['path']}")
  167. source['path'].unlink()
  168. media.remove(source)
  169. except BaseException as e:
  170. logger.error(f"Unlink error: {e}")
  171. continue
  172. # Rename
  173. err = None
  174. try:
  175. logger.debug(f"Renaming: {temp.name} -> {final.name}")
  176. temp.rename(final)
  177. video['path'] = final
  178. video['name'] = final.name
  179. media.append(video)
  180. except BaseException as e:
  181. err = e
  182. if not final.exists():
  183. err = 'Output file missing'
  184. elif final.stat().st_size < (min_size * size):
  185. err = 'Output file too small'
  186. if err is not None:
  187. logger.error(f"Rename error ({final}): {e}")
  188. continue
  189. def lang(media):
  190. logger.debug(f"Language processing {len(media)} files")
  191. for file in media:
  192. logger.debug(f"Lang: {file['name']}")
  193. # Skip files tha do not need lang processing
  194. if 'lang_vers' in file and file['lang_vers'] and file['lang_vers'] >= lang_vers:
  195. logger.debug(f"Skipping item with current lang_vers: {file['path']}")
  196. continue
  197. # Probe streams
  198. streams = probe.probe(file['path'], logger=logger)
  199. # Find missing and undefined languages for subtitle and audio streams
  200. updates = []
  201. for stream in streams:
  202. if all(name in stream for name in [ 'codec_type', 'index' ]) and stream['codec_type'] in [ 'subtitle', 'audio' ]:
  203. if 'TAG:language' not in stream or stream['TAG:language'] in [ None, 'und' ]:
  204. logger.debug(f"Missing or undefined language in {file['name']}:{stream['codec_type'][0]}:{stream['index']}")
  205. updates.append(int(stream['index']))
  206. # mkvpropedit
  207. if updates:
  208. args = [ propedit_exec, file['path'] ]
  209. for index in updates:
  210. # mkvpropedit starts counting at 1
  211. args.extend([ '--edit', f"track:{index + 1}", '--set', f"language={lang_default}" ])
  212. logger.debug(f"propedit command: {args}")
  213. propedit = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  214. logger.debug(f"propedit result: {propedit}")
  215. if propedit.returncode != 0:
  216. logger.error(f"ffprobe error ({propedit.returncode}): {propedit.stdout.decode('utf-8')}")
  217. sys.exit(propedit.returncode)
  218. # Set xattr
  219. if not write_xattr(file['path'], lang_xattr, lang_vers):
  220. continue
  221. file['lang_vers'] = lang_vers
  222. # Execute
  223. for (parent, dirs, files) in root.walk():
  224. logger.debug(f"Walking: {parent}")
  225. media = scan(files)
  226. mux(media)
  227. lang(media)