Home | History | Annotate | Download | only in tests 
      1 #!/usr/bin/env python3
      2 """Test LZ4 interoperability between versions"""
      3 
      4 #
      5 # Copyright (C) 2011-present, Takayuki Matsuoka
      6 # All rights reserved.
      7 # GPL v2 License
      8 #
      9 
     10 import glob
     11 import subprocess
     12 import filecmp
     13 import os
     14 import shutil
     15 import sys
     16 import hashlib
     17 
     18 repo_url = 'https://github.com/lz4/lz4.git'
     19 tmp_dir_name = 'tests/versionsTest'
     20 make_cmd = 'make'
     21 git_cmd = 'git'
     22 test_dat_src = 'README.md'
     23 test_dat = 'test_dat'
     24 head = 'v999'
     25 
     26 def proc(cmd_args, pipe=True, dummy=False):
     27     if dummy:
     28         return
     29     if pipe:
     30         subproc = subprocess.Popen(cmd_args,
     31                                    stdout=subprocess.PIPE,
     32                                    stderr=subprocess.PIPE)
     33     else:
     34         subproc = subprocess.Popen(cmd_args)
     35     return subproc.communicate()
     36 
     37 def make(args, pipe=True):
     38     return proc([make_cmd] + args, pipe)
     39 
     40 def git(args, pipe=True):
     41     return proc([git_cmd] + args, pipe)
     42 
     43 def get_git_tags():
     44     stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]'])
     45     tags = stdout.decode('utf-8').split()
     46     stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]'])
     47     tags += stdout.decode('utf-8').split()
     48     return tags
     49 
     50 # https://stackoverflow.com/a/19711609/2132223
     51 def sha1_of_file(filepath):
     52     with open(filepath, 'rb') as f:
     53         return hashlib.sha1(f.read()).hexdigest()
     54 
     55 if __name__ == '__main__':
     56     error_code = 0
     57     base_dir = os.getcwd() + '/..'           # /path/to/lz4
     58     tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/lz4/tests/versionsTest
     59     clone_dir = tmp_dir + '/' + 'lz4'        # /path/to/lz4/tests/versionsTest/lz4
     60     programs_dir = base_dir + '/programs'    # /path/to/lz4/programs
     61     os.makedirs(tmp_dir, exist_ok=True)
     62 
     63     # since Travis clones limited depth, we should clone full repository
     64     if not os.path.isdir(clone_dir):
     65         git(['clone', repo_url, clone_dir])
     66 
     67     shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
     68 
     69     # Retrieve all release tags
     70     print('Retrieve all release tags :')
     71     os.chdir(clone_dir)
     72     tags = [head] + get_git_tags()
     73     print(tags);
     74 
     75     # Build all release lz4c and lz4c32
     76     for tag in tags:
     77         os.chdir(base_dir)
     78         dst_lz4c   = '{}/lz4c.{}'  .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c.<TAG>
     79         dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32.<TAG>
     80         if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32) or tag == head:
     81             if tag != head:
     82                 r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/lz4/test/lz4test/<TAG>
     83                 os.makedirs(r_dir, exist_ok=True)
     84                 os.chdir(clone_dir)
     85                 git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
     86                 os.chdir(r_dir + '/programs')  # /path/to/lz4/lz4test/<TAG>/programs
     87             else:
     88                 os.chdir(programs_dir)
     89             make(['clean', 'lz4c'], False)
     90             shutil.copy2('lz4c',   dst_lz4c)
     91             make(['clean', 'lz4c32'], False)
     92             shutil.copy2('lz4c32', dst_lz4c32)
     93 
     94     # Compress test.dat by all released lz4c and lz4c32
     95     print('Compress test.dat by all released lz4c and lz4c32')
     96     os.chdir(tmp_dir)
     97     for lz4 in glob.glob("*.lz4"):
     98         os.remove(lz4)
     99     for tag in tags:
    100         proc(['./lz4c.'   + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4'])
    101         proc(['./lz4c.'   + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4'])
    102         proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4'])
    103         proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4'])
    104 
    105     print('Full list of compressed files')
    106     lz4s = sorted(glob.glob('*.lz4'))
    107     for lz4 in lz4s:
    108         print(lz4 + ' : ' + repr(os.path.getsize(lz4)))
    109 
    110     # Remove duplicated .lz4 files
    111     print('')
    112     print('Duplicated files')
    113     lz4s = sorted(glob.glob('*.lz4'))
    114     for i, lz4 in enumerate(lz4s):
    115         if not os.path.isfile(lz4):
    116             continue
    117         for j in range(i+1, len(lz4s)):
    118             lz4t = lz4s[j]
    119             if not os.path.isfile(lz4t):
    120                 continue
    121             if filecmp.cmp(lz4, lz4t):
    122                 os.remove(lz4t)
    123                 print('{} == {}'.format(lz4, lz4t))
    124 
    125     print('Enumerate only different compressed files')
    126     lz4s = sorted(glob.glob('*.lz4'))
    127     for lz4 in lz4s:
    128         print(lz4 + ' : ' + repr(os.path.getsize(lz4)) + ', ' + sha1_of_file(lz4))
    129 
    130     # Decompress remained .lz4 files by all released lz4c and lz4c32
    131     print('Decompression tests and verifications')
    132     lz4s = sorted(glob.glob('*.lz4'))
    133     for dec in glob.glob("*.dec"):
    134         os.remove(dec)
    135     for lz4 in lz4s:
    136         print(lz4, end=" ")
    137         for tag in tags:
    138             print(tag, end=" ")
    139             proc(['./lz4c.'   + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec'])
    140             proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec'])
    141         print(' OK')   # well, here, decompression has worked; but file is not yet verified
    142 
    143     # Compare all '.dec' files with test_dat
    144     decs = glob.glob('*.dec')
    145     for dec in decs:
    146         if not filecmp.cmp(dec, test_dat):
    147             print('ERR : ' + dec)
    148             error_code = 1
    149         else:
    150             print('OK  : ' + dec)
    151             os.remove(dec)
    152 
    153     if error_code != 0:
    154         print('ERROR')
    155 
    156     sys.exit(error_code)
    157