我的项目需要使用mksqashfs将某些东西打包到图像中，当内容更改时，我需要用新的文件替换图像文件。
使用SQASHFS之前，I'V使用了Tarball MD5SUM。内容变化，MD5改变。但是，Mksquashfs导出图像文件每次都会更改，即使内容相同。
我正在浏览Mksquashfs的所有参数，请尝试在没有运气的情况下找到问题。
github上还有另一个修改的mksquashf：
https://github.com/dannyp11/squashfs4.3.git
此叉子添加-no -state选项，该选项使Mksquashfs创建壁球文件，而无需日期信息。因此，如果内容保持不变，则结果南瓜文件具有相同的chksum。
diff -ruNa squashfs4.3/README squashfs4.3-new/README
--- squashfs4.3/README 2014-05-12 13:17:59.000000000 -0700
+++ squashfs4.3-new/README 2017-08-01 23:05:44.428051586 -0700
@@ -127,6 +127,9 @@
-keep-as-directory if one source directory is specified, create a root
directory containing that directory, rather than the
contents of the directory
+-no-date do not store the date in the squash file
+ (-noappend and -no-fragments is also activated)
+ this way, the squash always has the same cksum
Filesystem filter options:
-p <pseudo-definition> Add pseudo file definition
diff -ruNa squashfs4.3/squashfs-tools/mksquashfs.c squashfs4.3-new/squashfs-tools/mksquashfs.c
--- squashfs4.3/squashfs-tools/mksquashfs.c 2014-05-12 15:18:20.000000000 -0700
+++ squashfs4.3-new/squashfs-tools/mksquashfs.c 2017-08-01 21:26:03.092224142 -0700
@@ -98,6 +98,7 @@
int use_regex = FALSE;
int nopad = FALSE;
int exit_on_error = FALSE;
+int no_date = FALSE;
long long global_uid = -1, global_gid = -1;
@@ -3139,7 +3140,7 @@
buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
buf.st_uid = getuid();
buf.st_gid = getgid();
- buf.st_mtime = time(NULL);
+ buf.st_mtime = (!no_date)? time(NULL) : 0;
buf.st_dev = 0;
buf.st_ino = 0;
dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0);
@@ -3526,7 +3527,7 @@
buf.st_gid = pseudo_ent->dev->gid;
buf.st_rdev = makedev(pseudo_ent->dev->major,
pseudo_ent->dev->minor);
- buf.st_mtime = time(NULL);
+ buf.st_mtime = (!no_date)? time(NULL) : 0;
buf.st_ino = pseudo_ino ++;
if(pseudo_ent->dev->type == 'd') {
@@ -5262,6 +5263,12 @@
else if(strcmp(argv[i], "-keep-as-directory") == 0)
keep_as_directory = TRUE;
+ else if (strcmp(argv[i], "-no-date") == 0) {
+ no_date = TRUE;
+ delete = TRUE;
+ no_fragments = TRUE;
+ }
+
else if(strcmp(argv[i], "-exit-on-error") == 0)
exit_on_error = TRUE;
@@ -5315,6 +5322,9 @@
ERROR("tttdirectory containing that directory, "
"rather than then");
ERROR("tttcontents of the directoryn");
+ ERROR("-no-datettdo not store the date in the squash filen");
+ ERROR("ttt(-noappend and -no-fragments is also activated)n");
+ ERROR("tttthis way, the squash always has the same cksumn");
ERROR("nFilesystem filter options:n");
ERROR("-p <pseudo-definition>tAdd pseudo file "
"definitionn");
@@ -5717,7 +5727,7 @@
sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
always_use_fragments, duplicate_checking, exportable,
no_xattrs, comp_opts);
- sBlk.mkfs_time = time(NULL);
+ sBlk.mkfs_time = (!no_date) ? time(NULL) : 0;
disable_info();
我也有同样的问题，在塞拉克(Serach(之后，我发现了这一点：
＃32请求向Mksquashfs添加确定性行为
Here is a patch.
The patch adds "-deterministic" option.
Here is what the option does :
- it forces to one processor
- it forces the date globaly. The forced date is printed out and must be reused with "-forcedate" option in order to reproduced the same squashfs image.
- it sorts entries of directory as they are read.
- it delays the start of the fragment deflator thread. When this late start is active, the queue to fragment deflator thread is using an implementation of the queue with unlimited size.
Entries of directory must be sorted because the inode numbers are attributed as the files are discovered ("lookup_inode()" in "dirscan1()").
The format of the date after "-forcedate" is the number of seconds since the Epoch : the output of "date '+%s'".
With this patch, I can copy a tree of files, I can use tar or I can repack a mounted squashfs, and always have the same suqashfs file.
--- a/squashfs-tools/mksquashfs.c
+++ b/squashfs-tools/mksquashfs.c
@@ -113,6 +113,9 @@
#include "compressor.h"
#include "xattr.h"
+int forcedate = -1;
+int sort_dir_entry = 0;
+
int delete = FALSE;
int fd;
int cur_uncompressed = 0, estimated_uncompressed = 0;
@@ -342,6 +345,18 @@ struct queue {
void **data;
};
+/* struct describing queues used to pass data between threads */
+struct unlimited_queue {
+ struct unlimited_queue_node *first;
+ struct unlimited_queue_node *last;
+ pthread_mutex_t mutex;
+ pthread_cond_t any_event;
+};
+
+struct unlimited_queue_node {
+ void *data;
+ struct unlimited_queue_node* next;
+};
/* in memory uid tables */
#define ID_ENTRIES 256
@@ -362,6 +377,8 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
struct cache *reader_buffer, *writer_buffer, *fragment_buffer;
struct queue *to_reader, *from_reader, *to_writer, *from_writer, *from_deflate,
*to_frag;
+struct unlimited_queue *to_frag_unlimited = NULL;
+
pthread_t *thread, *deflator_thread, *frag_deflator_thread, progress_thread;
pthread_mutex_t fragment_mutex;
pthread_cond_t fragment_waiting;
@@ -488,6 +505,93 @@ void *queue_get(struct queue *queue)
return data;
}
+void queue_waitempty(struct queue *queue)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->readp != queue->writep)
+ pthread_cond_wait(&queue->full, &queue->mutex);
+
+ pthread_cond_signal(&queue->full);
+ pthread_mutex_unlock(&queue->mutex);
+}
+
+struct unlimited_queue *unlimited_queue_init()
+{
+ struct unlimited_queue *queue = malloc(sizeof(struct unlimited_queue));
+
+ if(queue == NULL)
+ goto failed;
+
+ queue->first = NULL;
+ queue->last = NULL;
+ pthread_mutex_init(&queue->mutex, NULL);
+ pthread_cond_init(&queue->any_event, NULL);
+
+ return queue;
+
+failed:
+ BAD_ERROR("Out of memory in queue_initn");
+}
+
+void unlimited_queue_put(struct unlimited_queue *queue, void *data)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ struct unlimited_queue_node *newNode = malloc(sizeof(struct unlimited_queue_node));
+ if(newNode == NULL){
+ BAD_ERROR("Out of memory in queue_initn");
+ }
+
+ newNode->data = data;
+ newNode->next = NULL;
+
+ pthread_mutex_unlock(&queue->mutex);
+
+ if(queue->first == NULL){
+ queue->first = newNode;
+ } else {
+ queue->last->next = newNode;
+ }
+ queue->last = newNode;
+ pthread_cond_broadcast(&queue->any_event);
+ pthread_mutex_unlock(&queue->mutex);
+}
+
+void *unlimited_queue_get(struct unlimited_queue *queue)
+{
+ void *data;
+
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->first == NULL)
+ pthread_cond_wait(&queue->any_event, &queue->mutex);
+
+ data = queue->first->data;
+ struct unlimited_queue_node* newFirst = queue->first->next;
+ free(queue->first);
+ queue->first = newFirst;
+/* eventually
+ if(newFirst == NULL){
+ queue->last == NULL;
+ }
+*/
+
+ pthread_cond_broadcast(&queue->any_event);
+ pthread_mutex_unlock(&queue->mutex);
+
+ return data;
+}
+
+void unlimited_queue_waitempty(struct unlimited_queue *queue)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->first != NULL)
+ pthread_cond_wait(&queue->any_event, &queue->mutex);
+
+ pthread_mutex_unlock(&queue->mutex);
+}
/* Cache status struct. Caches are used to keep
track of memory buffers passed between different threads */
@@ -1852,7 +1956,11 @@ void write_fragment()
fragment_data->block = fragments;
fragment_table[fragments].unused = 0;
fragments_outstanding ++;
- queue_put(to_frag, fragment_data);
+ if(to_frag_unlimited){
+ unlimited_queue_put(to_frag_unlimited, fragment_data);
+ }else{
+ queue_put(to_frag, fragment_data);
+ }
fragments ++;
fragment_size = 0;
pthread_mutex_unlock(&fragment_mutex);
@@ -2613,8 +2721,15 @@ void *frag_deflator(void *arg)
BAD_ERROR("frag_deflator:: compressor_init failedn");
while(1) {
+ struct file_buffer *file_buffer = NULL;
+
+ if(to_frag_unlimited){
+ file_buffer = unlimited_queue_get(to_frag_unlimited);
+ }else{
+ file_buffer = queue_get(to_frag);
+ }
+
int c_byte, compressed_size;
- struct file_buffer *file_buffer = queue_get(to_frag);
struct file_buffer *write_buffer =
cache_get(writer_buffer, file_buffer->block +
FRAG_INDEX, 1);
@@ -3594,7 +3709,11 @@ void dir_scan(squashfs_inode *inode, char *pathname,
buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
buf.st_uid = getuid();
buf.st_gid = getgid();
- buf.st_mtime = time(NULL);
+ if(forcedate != -1){
+ buf.st_mtime = forcedate;
+ }else{
+ buf.st_mtime = time(NULL);
+ }
buf.st_dev = 0;
buf.st_ino = 0;
dir_ent->inode = lookup_inode(&buf);
@@ -3634,11 +3753,23 @@ void dir_scan(squashfs_inode *inode, char *pathname,
dir_ent->inode->type = SQUASHFS_DIR_TYPE;
}
+struct file_dir_name{
+ char* filename;
+ char* dirname;
+};
+
+int compare_file_dir_name(const void* a, const void* b){
+ return strcmp((*(struct file_dir_name**)a)->filename, (*(struct file_dir_name**)b)->filename);
+}
+
+#define BASE_ALLOC_INCREMENT 128
struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
int (_readdir)(char *, char *, struct dir_info *))
{
- char filename[8192], dir_name[8192];
+ char filename2[8192], dir_name2[8192];
+ char *filename = filename2;
+ char *dir_name = dir_name2;
struct dir_info *dir = scan1_opendir(pathname);
if(dir == NULL) {
@@ -3646,18 +3777,69 @@ struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
goto error;
}
- while(_readdir(filename, dir_name, dir) != FALSE) {
+ int base_count = 0;
+ void** base = NULL;
+
+ if(sort_dir_entry){
+ int base_allocated = BASE_ALLOC_INCREMENT;
+ base = malloc(base_allocated * sizeof(void*));
+
+ while(_readdir(filename2, dir_name2, dir) != FALSE) {
+ struct file_dir_name* new_element = malloc(sizeof(struct file_dir_name));
+ if(new_element == NULL){
+ BAD_ERROR("No more Memory !n");
+ }
+ new_element->filename = strdup(filename2);
+ if(new_element->filename == NULL){
+ BAD_ERROR("No more Memory !n");
+ }
+ new_element->dirname = strdup(dir_name2);
+ if(new_element->dirname == NULL){
+ BAD_ERROR("No more Memory !n");
+ }
+
+ if(base_count >= base_allocated){
+ base_allocated += BASE_ALLOC_INCREMENT;
+ base = realloc(base, base_allocated * sizeof(void*));
+ if(base == NULL){
+ BAD_ERROR("No more Memory !n");
+ }
+ }
+
+ base[base_count] = new_element;
+ ++base_count;
+ }
+
+ qsort(base, base_count, sizeof(void*), compare_file_dir_name);
+ }
+
+ int i = 0;
+
+ while(1){
+ if(sort_dir_entry){
+ if(i >= base_count){
+ break;
+ }
+ struct file_dir_name* cur_element = (struct file_dir_name*)base[i];
+ dir_name = cur_element->dirname;
+ filename = cur_element->filename;
+ }else{
+ if(_readdir(filename, dir_name, dir) == FALSE) {
+ break;
+ }
+ }
+
struct dir_info *sub_dir;
struct stat buf;
struct pathnames *new;
if(strcmp(dir_name, ".") == 0 || strcmp(dir_name, "..") == 0)
- continue;
+ goto loop_continue;
if(lstat(filename, &buf) == -1) {
ERROR("Cannot stat dir/file %s because %s, ignoring",
filename, strerror(errno));
- continue;
+ goto loop_continue;
}
if((buf.st_mode & S_IFMT) != S_IFREG &&
@@ -3669,29 +3851,40 @@ struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
(buf.st_mode & S_IFMT) != S_IFSOCK) {
ERROR("File %s has unrecognised filetype %d, ignoring"
"n", filename, buf.st_mode & S_IFMT);
- continue;
+ goto loop_continue;
}
if(old_exclude) {
if(old_excluded(filename, &buf))
- continue;
+ goto loop_continue;
} else {
if(excluded(paths, dir_name, &new))
- continue;
+ goto loop_continue;
}
if((buf.st_mode & S_IFMT) == S_IFDIR) {
sub_dir = dir_scan1(filename, new, scan1_readdir);
if(sub_dir == NULL)
- continue;
+ goto loop_continue;
dir->directory_count ++;
} else
sub_dir = NULL;
add_dir_entry(dir_name, filename, sub_dir, lookup_inode(&buf),
dir);
+ loop_continue:
+ if(sort_dir_entry){
+ free(dir_name);
+ free(filename);
+ free(base[i]);
+ base[i] = NULL;
+ i++;
+ }
}
+ if(sort_dir_entry){
+ free(base);
+ }
scan1_freedir(dir);
error:
@@ -3777,7 +3970,11 @@ struct dir_info *dir_scan2(struct dir_info *dir, struct pseudo *pseudo)
buf.st_gid = pseudo_ent->dev->gid;
buf.st_rdev = makedev(pseudo_ent->dev->major,
pseudo_ent->dev->minor);
- buf.st_mtime = time(NULL);
+ if(forcedate != -1){
+ buf.st_mtime = forcedate;
+ }else{
+ buf.st_mtime = time(NULL);
+ }
buf.st_ino = pseudo_ino ++;
if(pseudo_ent->dev->type == 'f') {
@@ -4039,6 +4236,7 @@ void add_old_root_entry(char *name, squashfs_inode inode, int inode_number,
old_root_entry[old_root_entries++].inode.root_entry = TRUE;
}
+int late_fragment_thread_start = 0;
void initialise_threads(int readb_mbytes, int writeb_mbytes,
int fragmentb_mbytes)
@@ -4046,7 +4244,7 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
int i;
sigset_t sigmask, old_mask;
int reader_buffer_size = readb_mbytes << (20 - block_log);
- int fragment_buffer_size = fragmentb_mbytes << (20 - block_log);
+ int fragment_buffer_size = (fragmentb_mbytes == -1 ? -1 : fragmentb_mbytes << (20 - block_log));
/*
* writer_buffer_size is global because it is needed in
@@ -4095,10 +4293,16 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
to_writer = queue_init(writer_buffer_size);
from_writer = queue_init(1);
from_deflate = queue_init(reader_buffer_size);
- to_frag = queue_init(fragment_buffer_size);
+
+ if(fragment_buffer_size == -1){
+ to_frag_unlimited = unlimited_queue_init();
+ }else{
+ to_frag = queue_init(fragment_buffer_size);
+ }
+
reader_buffer = cache_init(block_size, reader_buffer_size);
writer_buffer = cache_init(block_size, writer_buffer_size);
- fragment_buffer = cache_init(block_size, fragment_buffer_size);
+ fragment_buffer = cache_init(block_size, fragment_buffer_size != -1 ? fragment_buffer_size : FRAGMENT_BUFFER_DEFAULT << (20 - block_log));
pthread_create(&thread[0], NULL, reader, NULL);
pthread_create(&thread[1], NULL, writer, NULL);
pthread_create(&progress_thread, NULL, progress_thrd, NULL);
@@ -4109,9 +4313,11 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
if(pthread_create(&deflator_thread[i], NULL, deflator, NULL) !=
0)
BAD_ERROR("Failed to create threadn");
- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
- NULL) != 0)
- BAD_ERROR("Failed to create threadn");
+ if(!late_fragment_thread_start){
+ if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
+ NULL) != 0)
+ BAD_ERROR("Failed to create threadn");
+ }
}
printf("Parallel mksquashfs: Using %d processor%sn", processors,
@@ -4701,6 +4907,31 @@ int main(int argc, char *argv[])
"megabyte or largern", argv[0]);
exit(1);
}
+ } else if(strcmp(argv[i], "-forcedate") == 0) {
+ if(++i == argc) {
+ ERROR("%s: -forcedate missing fake daten", argv[0]);
+ exit(1);
+ }
+ forcedate = strtol(argv[i], &b, 10);
+ if(*b != ' ') {
+ ERROR("%s: -forcedate invalid parametern", argv[0]);
+ exit(1);
+ }
+ } else if(strcmp(argv[i], "-deterministic") == 0) {
+ if(forcedate == -1){
+ forcedate = time(NULL);
+ }
+ processors = 1;
+ late_fragment_thread_start = 1;
+ fragmentb_mbytes = -1;
+ if(sort_dir_entry == 0){
+ sort_dir_entry = 1;
+ }
+ } else if(strcmp(argv[i], "-late-fragment-thread-start") == 0) {
+ late_fragment_thread_start = 1;
+ fragmentb_mbytes = -1;
+ } else if(strcmp(argv[i], "-unlimited-fragment-queue") == 0) {
+ fragmentb_mbytes = -1;
} else if(strcmp(argv[i], "-b") == 0) {
if(++i == argc) {
ERROR("%s: -b missing block sizen", argv[0]);
@@ -5292,7 +5523,11 @@ printOptions:
sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
always_use_fragments, duplicate_checking, exportable,
no_xattrs, comp_opts);
- sBlk.mkfs_time = time(NULL);
+ if(forcedate != -1){
+ sBlk.mkfs_time = forcedate;
+ } else {
+ sBlk.mkfs_time = time(NULL);
+ }
restore_filesystem:
if(progress && estimated_uncompressed) {
@@ -5301,6 +5536,14 @@ restore_filesystem:
}
write_fragment();
+
+ if(late_fragment_thread_start){
+ queue_waitempty(from_reader);
+ if(pthread_create(&frag_deflator_thread[0], NULL, frag_deflator,
+ NULL) != 0)
+ BAD_ERROR("Failed to create threadn");
+ }
+
sBlk.fragments = fragments;
if(!restoring) {
unlock_fragments();
@@ -5340,7 +5583,8 @@ restore_filesystem:
write_destination(fd, SQUASHFS_START, sizeof(sBlk), &sBlk);
if(!nopad && (i = bytes & (4096 - 1))) {
- char temp[4096] = {0};
+ char temp[4096];
+ memset(&temp, 0, 4096);
write_destination(fd, bytes, 4096 - i, temp);
}
@@ -5401,6 +5645,9 @@ restore_filesystem:
printf("Number of directories %dn", dir_count);
printf("Number of ids (unique uids + gids) %dn", id_count);
printf("Number of uids %dn", uid_count);
+ if(forcedate != -1){
+ printf("forcedate was %dn", forcedate);
+ }
for(i = 0; i < id_count; i++) {
if(id_table[i]->flags & ISA_UID) {