path: root/recipes/squashfs-tools/files/squashfs3.1r2-tools-replace_zlib_with_lzma.patch

Modify the source code to use the lzma {,de}compression routines.

Leon Woestenberg <leonw@mailcan.com>

diff -Nur squashfs-tools/mksquashfs.c squashfs-lzma-tools/mksquashfs.c
--- squashfs-tools/mksquashfs.c	2006-08-31 00:07:37.000000000 +0200
+++ squashfs-lzma-tools/mksquashfs.c	2006-08-22 09:35:14.000000000 +0200
@@ -40,16 +40,13 @@
 #include <signal.h>
 #include <setjmp.h>
 #include <sys/mman.h>
-#include <pthread.h>
 
 #ifndef linux
 #define __BYTE_ORDER BYTE_ORDER
 #define __BIG_ENDIAN BIG_ENDIAN
 #define __LITTLE_ENDIAN LITTLE_ENDIAN
-#include <sys/sysctl.h>
 #else
 #include <endian.h>
-#include <sys/sysinfo.h>
 #endif
 
 #include <squashfs_fs.h>
@@ -149,7 +146,7 @@
 struct inode_info *inode_info[INODE_HASH_SIZE];
 
 /* hash tables used to do fast duplicate searches in duplicate check */
-struct file_info *dupl[65536];
+struct file_info *dupl[65536], *frag_dups[65536];
 int dup_files = 0;
 
 /* list of exclude dirs/files */
@@ -165,7 +162,7 @@
 
 /* fragment block data structures */
 int fragments = 0;
-struct file_buffer *fragment_data = NULL;
+char fragment_data[SQUASHFS_FILE_SIZE];
 int fragment_size = 0;
 struct fragment {
 	unsigned int		index;
@@ -174,7 +171,6 @@
 };
 #define FRAG_SIZE 32768
 squashfs_fragment_entry *fragment_table = NULL;
-int fragments_outstanding = 0;
 
 
 /* current inode number for directories and non directories */
@@ -198,15 +194,13 @@
 
 /* in memory file info */
 struct file_info {
-	long long		file_size;
 	long long		bytes;
 	unsigned short		checksum;
-	unsigned short		fragment_checksum;
 	long long		start;
 	unsigned int		*block_list;
 	struct file_info	*next;
 	struct fragment		*fragment;
-	char			checksum_flag;
+	unsigned short		fragment_checksum;
 };
 
 /* count of how many times SIGINT or SIGQUIT has been sent */
@@ -225,7 +219,6 @@
 	sdir_count, sfifo_count, ssock_count, sdup_files;
 int sfragments;
 int restore = 0;
-int threads;
 
 /* flag whether destination file is a block device */
 int block_device = 0;
@@ -236,67 +229,14 @@
 /* save destination file name for deleting on error */
 char *destination_file = NULL;
 
-/* data allocator status struct.  Allocators are used to keep
-  track of memory buffers passed between different threads */
-struct allocator {
-	int		max_buffers;
-	int		count;
-	int		buffer_size;
-	pthread_mutex_t	mutex;
-	pthread_cond_t	wait;
-};
-
-/* struct describing a memory buffer passed between threads */
-struct file_buffer {
-	struct allocator	*allocator;
-	void			(*release)(int);
-	int			release_data;
-	long long		block;
-	int			size;
-	int			c_byte;
-	unsigned int		block_order;
-	int			fragment;
-	int			error;
-	struct file_buffer	*next;
-	char			data[0];
-};
-
-/* struct describing queues used to pass data between threads */
-struct queue {
-	int			size;
-	int			readp;
-	int			writep;
-	pthread_mutex_t		mutex;
-	pthread_cond_t		empty;
-	pthread_cond_t		full;
-	void			**data;
-};
-
-/* describes the list of blocks in a file which is a possible
-   duplicate.  For each block, it indicates whether the block is
-   in memory or on disk */
-struct buffer_list {
-	long long start;
-	int size;
-	struct file_buffer *read_buffer;
+/* structure to used to pass in a pointer or an integer
+ * to duplicate buffer read helper functions.
+ */
+struct duplicate_buffer_handle {
+	char	*ptr;
+	long long	start;
 };
 
-struct allocator *reader_buffer, *writer_buffer, *fragment_buffer;
-struct queue *to_reader, *from_reader, *to_writer, *from_writer, *from_deflate, *to_frag;
-pthread_t *thread, *deflator_thread, *frag_deflator_thread;
-pthread_mutex_t	fragment_mutex;
-pthread_cond_t fragment_waiting;
-pthread_mutex_t	pos_mutex;
-
-/* user options that control parallelisation */
-int processors = -1;
-/* default size of output buffer in Mbytes */
-#define WRITER_BUFFER_DEFAULT 512
-/* default size of input buffer in Mbytes */
-#define READER_BUFFER_DEFAULT 64
-int writer_buffer_size;
-int reader_buffer_size;
-
 void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, int type);
 extern int read_super(int fd, squashfs_super_block *sBlk, int *be, char *source);
 extern long long read_filesystem(char *root_name, int fd, squashfs_super_block *sBlk, char **cinode_table,
@@ -309,166 +249,18 @@
 		unsigned int *inode_dir_inode_number, unsigned int *inode_dir_parent_inode,
 		void (push_directory_entry)(char *, squashfs_inode, int, int),
 		squashfs_fragment_entry **fragment_table);
-extern int read_sort_file(char *filename, int source, char *source_path[]);
-extern void sort_files_and_write(struct dir_info *dir);
-struct file_info *duplicate(long long file_size, long long bytes, unsigned int **block_list, long long *start, struct fragment **fragment, struct file_buffer *file_buffer, struct buffer_list *buffer_list, int blocks, unsigned short checksum, unsigned short fragment_checksum, int checksum_flag);
+int get_sorted_inode(squashfs_inode *inode, struct stat *buf);
+int read_sort_file(char *filename, int source, char *source_path[]);
+void sort_files_and_write(struct dir_info *dir);
+struct file_info *duplicate(char *(get_next_file_block)(struct duplicate_buffer_handle *, unsigned int), struct duplicate_buffer_handle *file_start, long long bytes, unsigned int **block_list, long long *start, int blocks, struct fragment **fragment, char *frag_data, int frag_bytes);
 struct dir_info *dir_scan1(char *, int (_readdir)(char *, char *, struct dir_info *));
-struct file_info *add_non_dup(long long file_size, long long bytes, unsigned int *block_list, long long start, struct fragment *fragment, unsigned short checksum, unsigned short fragment_checksum, int checksum_flag);
-extern void generate_file_priorities(struct dir_info *dir, int priority, struct stat *buf);
-extern struct priority_entry *priority_list[65536];
-
-
-struct allocator *alloc_init(int buffer_size, int max_buffers)
-{
-	struct allocator *allocator = malloc(sizeof(struct allocator));
-
-	if(allocator == NULL)
-		return NULL;
-
-	allocator->max_buffers = max_buffers;
-	allocator->buffer_size = buffer_size;
-	allocator->count = 0;
-	pthread_mutex_init(&allocator->mutex, NULL);
-	pthread_cond_init(&allocator->wait, NULL);
-
-	return allocator;
-}
-
-
-struct file_buffer *alloc_get(struct allocator *allocator)
-{
-	struct file_buffer *file_buffer;
-	
-	pthread_mutex_lock(&allocator->mutex);
-
-	while(allocator->count == allocator->max_buffers)
-		pthread_cond_wait(&allocator->wait, &allocator->mutex);
-
-	if((file_buffer = malloc(sizeof(struct file_buffer) + allocator->buffer_size)) == NULL)
-		goto failed;
-
-	file_buffer->release = NULL;
-	file_buffer->allocator = allocator;
-	allocator->count ++;
-
-failed:
-	pthread_mutex_unlock(&allocator->mutex);
-	return file_buffer;
-}
-
-
-struct file_buffer *alloc_get_2(struct allocator *allocator, void (*release)(int), int release_data)
-{
-	struct file_buffer *file_buffer = alloc_get(allocator);
-
-	if(file_buffer) {
-		file_buffer->release = release;
-		file_buffer->release_data = release_data;
-	}
-	return file_buffer;
-}
-
-
-void alloc_free(struct file_buffer *file_buffer)
-{
-	struct allocator *allocator;
-
-	if(file_buffer == NULL)
-		return;
-
-	allocator = file_buffer->allocator;
-
-	if(file_buffer->release)
-		file_buffer->release(file_buffer->release_data);
-
-	pthread_mutex_lock(&allocator->mutex);
-	free(file_buffer);
-	if(allocator->count == 0)
-		ERROR("alloc_free: freeing buffer for empty allocator!\n");
-	else
-		allocator->count --;
-	pthread_cond_signal(&allocator->wait);
-	pthread_mutex_unlock(&allocator->mutex);
-}
-
-
-struct queue *queue_init(int size)
-{
-	struct queue *queue = malloc(sizeof(struct queue));
-
-	if(queue == NULL)
-		return NULL;
-
-	if((queue->data = malloc(sizeof(void *) * (size + 1))) == NULL) {
-		free(queue);
-		return NULL;
-	}
-
-	queue->size = size + 1;
-	queue->readp = queue->writep = 0;
-	pthread_mutex_init(&queue->mutex, NULL);
-	pthread_cond_init(&queue->empty, NULL);
-	pthread_cond_init(&queue->full, NULL);
-
-	return queue;
-}
-
-
-void queue_put(struct queue *queue, void *data)
-{
-	int nextp;
-
-	pthread_mutex_lock(&queue->mutex);
-
-	while((nextp = (queue->writep + 1) % queue->size) == queue->readp)
-		pthread_cond_wait(&queue->full, &queue->mutex);
-
-	queue->data[queue->writep] = data;
-	queue->writep = nextp;
-	pthread_cond_signal(&queue->empty);
-	pthread_mutex_unlock(&queue->mutex);
-}
-
-
-void *queue_get(struct queue *queue)
-{
-	void *data;
-	pthread_mutex_lock(&queue->mutex);
-
-	while(queue->readp == queue->writep)
-		pthread_cond_wait(&queue->empty, &queue->mutex);
-
-	data = queue->data[queue->readp];
-	queue->readp = (queue->readp + 1) % queue->size;
-	pthread_cond_signal(&queue->full);
-	pthread_mutex_unlock(&queue->mutex);
-
-	return data;
-}
-
 
 #define MKINODE(A)	((squashfs_inode)(((squashfs_inode) inode_bytes << 16) + (((char *)A) - data_cache)))
 
 
-inline void waitforthread(int i)
-{
-	TRACE("Waiting for thread %d\n", i);
-	while(thread[i] != 0)
-		sched_yield();
-}
-
-
 void restorefs()
 {
-	int i;
-
 	ERROR("Exiting - restoring original filesystem!\n\n");
-
-	for(i = 0; i < 2 + processors * 2; i++)
-		pthread_kill(thread[i], SIGUSR1);
-	for(i = 0; i < 2 + processors * 2; i++)
-		waitforthread(i);
-	TRACE("All threads in signal handler\n");
 	bytes = sbytes;
 	memcpy(data_cache, sdata_cache, cache_bytes = scache_bytes);
 	memcpy(directory_data_cache, sdirectory_data_cache, directory_cache_bytes = sdirectory_cache_bytes);
@@ -495,13 +287,12 @@
 
 void sighandler()
 {
-	if(++interrupted > 2)
-		return;
-	if(interrupted == 2)
+	if(interrupted == 1)
 		restorefs();
 	else {
 		ERROR("Interrupting will restore original filesystem!\n");
 		ERROR("Interrupt again to quit\n");
+		interrupted ++;
 	}
 }
 
@@ -512,81 +303,22 @@
 }
 
 
-void sigusr1_handler()
-{
-	int i;
-	sigset_t sigmask;
-	pthread_t thread_id = pthread_self();
-
-	for(i = 0; i < (2 + processors * 2) && thread[i] != thread_id; i++);
-	thread[i] = (pthread_t) 0;
-
-	TRACE("Thread %d(%p) in sigusr1_handler\n", i, thread_id);
-
-	sigemptyset(&sigmask);
-	sigaddset(&sigmask, SIGINT);
-	sigaddset(&sigmask, SIGQUIT);
-	sigaddset(&sigmask, SIGUSR1);
-	while(1) {
-		sigsuspend(&sigmask);
-		TRACE("After wait in sigusr1_handler :(\n");
-	}
-}
-
-
-unsigned int mangle2(z_stream **strm, char *d, char *s, int size, int block_size, int uncompressed, int data_block)
+unsigned int mangle(char *d, char *s, int size, int block_size, int uncompressed, int data_block)
 {
-	unsigned long c_byte;
+	unsigned long c_byte = block_size << 1;
 	unsigned int res;
-	z_stream *stream = *strm;
-
-	if(uncompressed)
-		goto notcompressed;
-
-	if(stream == NULL) {
-		if((stream = *strm = malloc(sizeof(z_stream))) == NULL)
-			BAD_ERROR("mangle::compress failed, not enough memory\n");
-
-		stream->zalloc = Z_NULL;
-		stream->zfree = Z_NULL;
-		stream->opaque = 0;
 
-		if((res = deflateInit(stream, 9)) != Z_OK) {
-			if(res == Z_MEM_ERROR)
-				BAD_ERROR("zlib::compress failed, not enough memory\n");
-			else if(res == Z_STREAM_ERROR)
-				BAD_ERROR("zlib::compress failed, not a valid compression level\n");
-			else if(res == Z_VERSION_ERROR)
-				BAD_ERROR("zlib::compress failed, incorrect zlib version\n");
-			else
-				BAD_ERROR("zlib::compress failed, unknown error %d\n", res);
-		}
-	} else if((res = deflateReset(stream)) != Z_OK) {
-		if(res == Z_STREAM_ERROR)
-			BAD_ERROR("zlib::compress failed, stream state inconsistent\n");
-		else
-			BAD_ERROR("zlib::compress failed, unknown error %d\n", res);
-	}
-
-	stream->next_in = (unsigned char *) s;
-	stream->avail_in = size;
-	stream->next_out = (unsigned char *) d;
-	stream->avail_out = block_size;
-
-	res = deflate(stream, Z_FINISH);
-	if(res != Z_STREAM_END && res != Z_OK) {
-		if(res == Z_STREAM_ERROR)
-			BAD_ERROR("zlib::compress failed, stream state inconsistent\n");
+	if(!uncompressed && (res = compress2((unsigned char *) d, &c_byte, (unsigned char *) s, size, 9)) != Z_OK) {
+		if(res == Z_MEM_ERROR)
+			BAD_ERROR("zlib::compress failed, not enough memory\n");
 		else if(res == Z_BUF_ERROR)
-			BAD_ERROR("zlib::compress failed, no progress possible\n");
+			BAD_ERROR("zlib::compress failed, not enough room in output buffer\n");
 		else
 			BAD_ERROR("zlib::compress failed, unknown error %d\n", res);
+		return 0;
 	}
 
-	c_byte = stream->total_out;
-
-	if(res != Z_STREAM_END || c_byte >= size) {
-notcompressed:
+	if(uncompressed || c_byte >= size) {
 		memcpy(d, s, size);
 		return size | (data_block ? SQUASHFS_COMPRESSED_BIT_BLOCK : SQUASHFS_COMPRESSED_BIT);
 	}
@@ -595,14 +327,6 @@
 }
 
 
-unsigned int mangle(char *d, char *s, int size, int block_size, int uncompressed, int data_block)
-{
-	static z_stream *stream = NULL;
-
-	return mangle2(&stream, d, s, size, block_size, uncompressed, data_block);
-}
-
-
 squashfs_base_inode_header *get_inode(int req_size)
 {
 	int data_space;
@@ -655,7 +379,6 @@
 {
 	off_t off = byte;
 
-	pthread_mutex_lock(&pos_mutex);
 	if(lseek(fd, off, SEEK_SET) == -1) {
 		perror("Lseek on destination failed");
 		EXIT_MKSQUASHFS();
@@ -665,7 +388,6 @@
 		perror("Read on destination failed");
 		EXIT_MKSQUASHFS();
 	}
-	pthread_mutex_unlock(&pos_mutex);
 }
 
 
@@ -673,9 +395,6 @@
 {
 	off_t off = byte;
 
-	if(interrupted < 2)
-		pthread_mutex_lock(&pos_mutex);
-
 	if(lseek(fd, off, SEEK_SET) == -1) {
 		perror("Lseek on destination failed");
 		EXIT_MKSQUASHFS();
@@ -685,9 +404,6 @@
 		perror("Write on destination failed");
 		EXIT_MKSQUASHFS();
 	}
-	
-	if(interrupted < 2)
-		pthread_mutex_unlock(&pos_mutex);
 }
 
 
@@ -1173,23 +889,10 @@
 }
 
 
-char *get_fragment(char *buffer, struct fragment *fragment, int *cached_fragment)
+char *get_fragment(char *buffer, struct fragment *fragment)
 {
-	squashfs_fragment_entry *disk_fragment;
-	int size;
-
-	if(fragment->index == *cached_fragment || fragment->index == SQUASHFS_INVALID_FRAG)
-		return buffer + fragment->offset;
-
-	if(fragment_data && fragment->index == fragments)
-		return fragment_data->data + fragment->offset;
-
-	pthread_mutex_lock(&fragment_mutex);
-	while(fragment_table[fragment->index].pending)
-		pthread_cond_wait(&fragment_waiting, &fragment_mutex);
-	pthread_mutex_unlock(&fragment_mutex);
-	disk_fragment = &fragment_table[fragment->index];
-	size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size);
+	squashfs_fragment_entry *disk_fragment = &fragment_table[fragment->index];
+	int size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size);
 
 	if(SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) {
 		int res;
@@ -1209,74 +912,53 @@
 	} else
 		read_bytes(fd, disk_fragment->start_block, size, buffer);
 
-	*cached_fragment = fragment->index;
 	return buffer + fragment->offset;
 }
 
 	
-void ensure_fragments_flushed()
-{
-	pthread_mutex_lock(&fragment_mutex);
-	while(fragments_outstanding)
-		pthread_cond_wait(&fragment_waiting, &fragment_mutex);
-	pthread_mutex_unlock(&fragment_mutex);
-}
-
-
 void write_fragment()
 {
+	int compressed_size;
+	char buffer[block_size << 1];
+
 	if(fragment_size == 0)
 		return;
 
-	pthread_mutex_lock(&fragment_mutex);
-	if(fragments % FRAG_SIZE == 0) {
-		if((fragment_table = (squashfs_fragment_entry *) realloc(fragment_table, (fragments + FRAG_SIZE) * sizeof(squashfs_fragment_entry))) == NULL) {
-			pthread_mutex_unlock(&fragment_mutex);
+	if(fragments % FRAG_SIZE == 0)
+		if((fragment_table = (squashfs_fragment_entry *) realloc(fragment_table, (fragments + FRAG_SIZE) * sizeof(squashfs_fragment_entry))) == NULL)
 			BAD_ERROR("Out of memory in fragment table\n");
-		}
-	}
-	fragment_data->size = fragment_size;
-	fragment_data->block = fragments;
-	fragment_table[fragments].pending = TRUE;
-	fragments_outstanding ++;
-	queue_put(to_frag, fragment_data);
+	fragment_table[fragments].size = mangle(buffer, fragment_data, fragment_size, block_size, noF, 1);
+	fragment_table[fragments].start_block = bytes;
+	compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(fragment_table[fragments].size);
+	write_bytes(fd, bytes, compressed_size, buffer);
+	bytes += compressed_size;
+	total_uncompressed += fragment_size;
+	total_compressed += compressed_size;
+	TRACE("Writing fragment %d, uncompressed size %d, compressed size %d\n",fragments, fragment_size, compressed_size);
 	fragments ++;
 	fragment_size = 0;
-	pthread_mutex_unlock(&fragment_mutex);
-}
-
-void frag_release(int block)
-{
-	pthread_mutex_lock(&fragment_mutex);
-	fragment_table[block].pending = FALSE;
-	pthread_cond_signal(&fragment_waiting);
-	pthread_mutex_unlock(&fragment_mutex);
 }
 
 
 static struct fragment empty_fragment = {SQUASHFS_INVALID_FRAG, 0, 0};
-struct fragment *get_and_fill_fragment(struct file_buffer *file_buffer)
+struct fragment *get_and_fill_fragment(char *buff, int size)
 {
 	struct fragment *ffrg;
-	
 
-	if(file_buffer == NULL || file_buffer->size == 0)
+	if(size == 0)
 		return &empty_fragment;
 
-	if(fragment_size + file_buffer->size > block_size)
+	if(fragment_size + size > block_size)
 		write_fragment();
 
 	if((ffrg = (struct fragment *) malloc(sizeof(struct fragment))) == NULL)
 		BAD_ERROR("Out of memory in fragment block allocation!\n");
 
-	if(fragment_size == 0)
-		fragment_data = alloc_get(fragment_buffer);
-
 	ffrg->index = fragments;
 	ffrg->offset = fragment_size;
-	ffrg->size = file_buffer->size;
-	memcpy(fragment_data->data + fragment_size, file_buffer->data, file_buffer->size);
-	fragment_size += file_buffer->size;
+	ffrg->size = size;
+	memcpy(fragment_data + fragment_size, buff, size);
+	fragment_size += size;
 
 	return ffrg;
 }
@@ -1333,10 +1015,19 @@
 }
 
 
+char *read_from_buffer(struct duplicate_buffer_handle *handle, unsigned int avail_bytes)
+{
+	char *v = handle->ptr;
+	handle->ptr += avail_bytes;	
+	return v;
+}
+
+
 char read_from_file_buffer[SQUASHFS_FILE_MAX_SIZE];
-char *read_from_disk(long long start, unsigned int avail_bytes)
+char *read_from_file(struct duplicate_buffer_handle *handle, unsigned int avail_bytes)
 {
-	read_bytes(fd, start, avail_bytes, read_from_file_buffer);
+	read_bytes(fd, handle->start, avail_bytes, read_from_file_buffer);
+	handle->start += avail_bytes;
 	return read_from_file_buffer;
 }
 
@@ -1344,205 +1035,99 @@
 /*
  * Compute 16 bit BSD checksum over the data
  */
-unsigned short get_checksum(char *buff, int bytes, unsigned short chksum)
-{
-	unsigned char *b = (unsigned char *) buff;
-
-	while(bytes --) {
-		chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1;
-		chksum += *b++;
-	}
-
-	return chksum;
-}
-
-
-unsigned short get_checksum_disk(long long start, long long l)
+unsigned short get_checksum(char *(get_next_file_block)(struct duplicate_buffer_handle *, unsigned int), struct duplicate_buffer_handle *handle, long long l)
 {
 	unsigned short chksum = 0;
-	unsigned int bytes;
+	unsigned int bytes = 0;
+	unsigned char *b;
+	struct duplicate_buffer_handle position = *handle;
 
 	while(l) {
 		bytes = l > SQUASHFS_FILE_MAX_SIZE ? SQUASHFS_FILE_MAX_SIZE : l;
 		l -= bytes;
-		chksum = get_checksum(read_from_disk(start, bytes), bytes, chksum);
-		start += bytes;
-	}
-
-	return chksum;
-}
-
-
-unsigned short get_checksum_buffer(struct buffer_list *buffer_list, unsigned int blocks)
-{
-	unsigned short chksum = 0;
-	int block;
-
-	for(block = 0; block < blocks; block ++) {
-		struct buffer_list *b = &buffer_list[block];
-
-		if(b->read_buffer)
-			chksum = get_checksum(b->read_buffer->data, b->read_buffer->size, chksum);
-		else
-			chksum = get_checksum(read_from_disk(b->start, b->size), b->size, chksum);
+		b = (unsigned char *) get_next_file_block(&position, bytes);
+		while(bytes--) {
+			chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1;
+			chksum += *b++;
+		}
 	}
 
 	return chksum;
 }
 
 
-unsigned short get_checksum_mem(char *buff, int bytes)
-{
-	return get_checksum(buff, bytes, 0);
-}
-
-
-unsigned short get_checksum_mem_buffer(struct file_buffer *file_buffer)
-{
-	if(file_buffer == NULL)
-		return 0;
-	else
-		return get_checksum(file_buffer->data, file_buffer->size, 0);
-}
-
-
 int cached_frag = -1;
-char fragdata[SQUASHFS_FILE_MAX_SIZE];
-#define DUP_HASH(a) (a & 0xffff)
-void add_file(long long start, long long file_size, long long file_bytes, unsigned int *block_listp, int blocks, unsigned int fragment, int offset, int bytes)
+void add_file(long long start, long long file_bytes, unsigned int *block_listp, int blocks, unsigned int fragment, int offset, int bytes)
 {
 	struct fragment *frg;
+	struct file_info *dupl_ptr;
 	char *datap;
+	struct duplicate_buffer_handle handle;
 	unsigned int *block_list = block_listp;
-	struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
 
-	if(!duplicate_checking || file_size == 0)
+	if(!duplicate_checking)
 		return;
 
-	for(; dupl_ptr; dupl_ptr = dupl_ptr->next) {
-		if(file_size != dupl_ptr->file_size)
-			continue;
-		if(blocks != 0 && start != dupl_ptr->start)
-			continue;
-		if(fragment != dupl_ptr->fragment->index)
-			continue;
-		if(fragment != SQUASHFS_INVALID_FRAG && (offset != dupl_ptr->fragment->offset || bytes != dupl_ptr->fragment->size))
-			continue;
-		return;
-	}
-
 	if((frg = (struct fragment *) malloc(sizeof(struct fragment))) == NULL)
 		BAD_ERROR("Out of memory in fragment block allocation!\n");
 
 	frg->index = fragment;
 	frg->offset = offset;
 	frg->size = bytes;
-
-	add_non_dup(file_size, file_bytes, block_list, start, frg, 0, 0, FALSE);
+	if(fragment == cached_frag || fragment == SQUASHFS_INVALID_FRAG)
+		datap = fragment_data + offset;
+	else
+		datap = get_fragment(fragment_data, frg);
+	handle.start = start;
+	if((dupl_ptr = duplicate(read_from_file, &handle, file_bytes, &block_listp, &start, blocks, &frg, datap, bytes)) != NULL)
+		dupl_ptr->fragment = frg;
+	else
+		free(block_list);
+	cached_frag = fragment;
 }
 
+
 char cached_fragment[SQUASHFS_FILE_SIZE];
 int cached_frag1 = -1;
 
-int pre_duplicate(long long file_size)
-{
-	struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
-
-	for(; dupl_ptr; dupl_ptr = dupl_ptr->next)
-		if(dupl_ptr->file_size == file_size)
-			return TRUE;
-
-	return FALSE;
-}
-
-
-int pre_duplicate_frag(long long file_size, unsigned short checksum)
+struct file_info *duplicate(char *(get_next_file_block)(struct duplicate_buffer_handle *, unsigned int), struct duplicate_buffer_handle *file_start, long long bytes, unsigned int **block_list, long long *start, int blocks, struct fragment **fragment, char *frag_data, int frag_bytes)
 {
-	struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
-
-	for(; dupl_ptr; dupl_ptr = dupl_ptr->next)
-		if(dupl_ptr->file_size == file_size) {
-			if(dupl_ptr->checksum_flag == FALSE) {
-				dupl_ptr->checksum = get_checksum_disk(dupl_ptr->start, dupl_ptr->bytes);
-				dupl_ptr->fragment_checksum = get_checksum_mem(get_fragment(cached_fragment, dupl_ptr->fragment, &cached_frag1), file_size);
-				dupl_ptr->checksum_flag = TRUE;
-			}
-			if(dupl_ptr->fragment_checksum == checksum)
-				return TRUE;
-		}
-
-	return FALSE;
-}
-
-
-struct file_info *add_non_dup(long long file_size, long long bytes, unsigned int *block_list, long long start, struct fragment *fragment, unsigned short checksum, unsigned short fragment_checksum, int checksum_flag)
-{
-	struct file_info *dupl_ptr;
+	unsigned short checksum = get_checksum(get_next_file_block, file_start, bytes);
+	struct duplicate_buffer_handle handle = { frag_data, 0 };
+	unsigned short fragment_checksum = get_checksum(read_from_buffer, &handle, frag_bytes);
+	struct file_info *dupl_ptr = bytes ? dupl[checksum] : frag_dups[fragment_checksum];
 
-	if((dupl_ptr = (struct file_info *) malloc(sizeof(struct file_info))) == NULL) {
-		BAD_ERROR("Out of memory in dup_files allocation!\n");
-	}
-
-	dupl_ptr->file_size = file_size;
-	dupl_ptr->bytes = bytes;
-	dupl_ptr->block_list = block_list;
-	dupl_ptr->start = start;
-	dupl_ptr->fragment = fragment;
-	dupl_ptr->checksum = checksum;
-	dupl_ptr->fragment_checksum = fragment_checksum;
-	dupl_ptr->checksum_flag = checksum_flag;
-	dupl_ptr->next = dupl[DUP_HASH(file_size)];
-	dupl[DUP_HASH(file_size)] = dupl_ptr;
-	dup_files ++;
-
-	return dupl_ptr;
-}
-
-
-struct file_info *duplicate(long long file_size, long long bytes, unsigned int **block_list, long long *start, struct fragment **fragment, struct file_buffer *file_buffer, struct buffer_list *buffer_list, int blocks, unsigned short checksum, unsigned short fragment_checksum, int checksum_flag)
-{
-	struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
-	int frag_bytes = file_buffer ? file_buffer->size : 0;
 
 	for(; dupl_ptr; dupl_ptr = dupl_ptr->next)
-		if(file_size == dupl_ptr->file_size && bytes == dupl_ptr->bytes && frag_bytes == dupl_ptr->fragment->size) {
-			char buffer2[SQUASHFS_FILE_MAX_SIZE];
+		if(bytes == dupl_ptr->bytes && frag_bytes == dupl_ptr->fragment->size && fragment_checksum == dupl_ptr->fragment_checksum) {
+			char buffer1[SQUASHFS_FILE_MAX_SIZE];
+			long long dup_bytes = dupl_ptr->bytes;
 			long long dup_start = dupl_ptr->start;
+			struct duplicate_buffer_handle position = *file_start;
 			char *buffer;
-			int block;
+			while(dup_bytes) {
+				int avail_bytes = dup_bytes > SQUASHFS_FILE_MAX_SIZE ? SQUASHFS_FILE_MAX_SIZE : dup_bytes;
 
-			if(checksum_flag == FALSE) {
-				checksum = get_checksum_buffer(buffer_list, blocks);
-				fragment_checksum = get_checksum_mem_buffer(file_buffer);
-				checksum_flag = TRUE;
-			}
-
-			if(dupl_ptr->checksum_flag == FALSE) {
-				dupl_ptr->checksum = get_checksum_disk(dupl_ptr->start, dupl_ptr->bytes);
-				dupl_ptr->fragment_checksum = get_checksum_mem(get_fragment(cached_fragment, dupl_ptr->fragment, &cached_frag1), frag_bytes);
-				dupl_ptr->checksum_flag = TRUE;
-			}
-
-			if(checksum != dupl_ptr->checksum || fragment_checksum != dupl_ptr->fragment_checksum)
-				continue;
-
-			for(block = 0; block < blocks; block ++) {
-				struct buffer_list *b = &buffer_list[block];
-
-				if(b->read_buffer)
-					buffer = b->read_buffer->data;
-				else
-					buffer = read_from_disk(b->start, b->size);
-
-				read_bytes(fd, dup_start, b->size, buffer2);
-				if(memcmp(buffer, buffer2, b->size) != 0)
+				buffer = get_next_file_block(&position, avail_bytes);
+				read_bytes(fd, dup_start, avail_bytes, buffer1);
+				if(memcmp(buffer, buffer1, avail_bytes) != 0)
 					break;
-				dup_start += b->size;
+				dup_bytes -= avail_bytes;
+				dup_start += avail_bytes;
 			}
-			if(block == blocks) {
-				char *fragment_buffer1 = get_fragment(cached_fragment, dupl_ptr->fragment, &cached_frag1);
+			if(dup_bytes == 0) {
+				char *fragment_buffer1;
+				
+				if(dupl_ptr->fragment->index == fragments || dupl_ptr->fragment->index == SQUASHFS_INVALID_FRAG)
+					fragment_buffer1 = fragment_data + dupl_ptr->fragment->offset;
+				else if(dupl_ptr->fragment->index == cached_frag1)
+					fragment_buffer1 = cached_fragment + dupl_ptr->fragment->offset;
+				else {
+					fragment_buffer1 = get_fragment(cached_fragment, dupl_ptr->fragment);
+					cached_frag1 = dupl_ptr->fragment->index;
+				}
 
-				if(frag_bytes == 0 || memcmp(file_buffer->data, fragment_buffer1, frag_bytes) == 0) {
+				if(frag_bytes == 0 || memcmp(frag_data, fragment_buffer1, frag_bytes) == 0) {
 					TRACE("Found duplicate file, start 0x%llx, size %lld, checksum 0x%x, fragment %d, size %d, offset %d, checksum 0x%x\n", dupl_ptr->start,
 						dupl_ptr->bytes, dupl_ptr->checksum, dupl_ptr->fragment->index, frag_bytes, dupl_ptr->fragment->offset, fragment_checksum);
 					*block_list = dupl_ptr->block_list;
@@ -1554,520 +1139,135 @@
 		}
 
 
-	return add_non_dup(file_size, bytes, *block_list, *start, *fragment, checksum, fragment_checksum, checksum_flag);
-}
-
-
-void reader_read_file(struct dir_ent *dir_ent)
-{
-	struct stat *buf = &dir_ent->inode->buf;
-	int count;
-	int blocks = (buf->st_size + block_size - 1) >> block_log;
-	int frag_block = !no_fragments && (always_use_fragments ||
-		(buf->st_size < block_size)) ? buf->st_size >> block_log : -1;
-	int file;
-	static int block_order = 0;
-	struct file_buffer *file_buffer;
-
-	if(buf->st_size == 0 || dir_ent->inode->read)
-		return;
-
-	if((file = open(dir_ent->pathname, O_RDONLY)) == -1)
-		goto read_err;
-
-	for(count = 0; count < blocks; count ++) {
-		file_buffer = alloc_get(reader_buffer);
-
-		if((file_buffer->size = read(file, file_buffer->data, block_size)) == -1) {
-			close(file);
-			goto read_err2;
-		}
-		file_buffer->block = count;
-		file_buffer->block_order = block_order ++;
-		file_buffer->error = FALSE;
-		if(file_buffer->fragment = count == frag_block)
-			queue_put(from_deflate, file_buffer);
-		else
-			queue_put(from_reader, file_buffer);
-	}
-
-	close(file);
-	dir_ent->inode->read = TRUE;
-
-	return;
-
-read_err:
-	file_buffer = alloc_get(reader_buffer);
-read_err2:
-	file_buffer->block_order = block_order ++;
-	file_buffer->error = TRUE;
-	queue_put(from_deflate, file_buffer);
-}
-
-
-void reader_scan(struct dir_info *dir) {
-	int i;
-
-	for(i = 0; i < dir->count; i++) {
-		struct dir_ent *dir_ent = dir->list[i];
-		struct stat *buf = &dir_ent->inode->buf;
-		if(dir_ent->data)
-			continue;
-
-		switch(buf->st_mode & S_IFMT) {
-			case S_IFREG:
-				reader_read_file(dir_ent);
-				break;
-			case S_IFDIR:
-				reader_scan(dir_ent->dir);
-				break;
-		}
-	}
-}
-
-
-void *reader(void *arg)
-{
-	int oldstate;
-
-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
-	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate);
-
-	if(!sorted)
-		reader_scan(queue_get(to_reader));
-	else {
-		int i;
-		struct priority_entry *entry;
-
-		queue_get(to_reader);
-		for(i = 65535; i >= 0; i--)
-			for(entry = priority_list[i]; entry; entry = entry->next)
-				reader_read_file(entry->dir);
-	}
-}
-
-
-void *writer(void *arg)
-{
-	int write_error = FALSE;
-	int oldstate;
-
-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
-	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate);
-
-	while(1) {
-		struct file_buffer *file_buffer = queue_get(to_writer);
-		off_t off;
-
-		if(file_buffer == NULL) {
-			queue_put(from_writer, write_error ? (void *) &write_error : NULL);
-			continue;
-		}
-
-		off = file_buffer->block;
-
-		pthread_mutex_lock(&pos_mutex);
-
-		if(!write_error && lseek(fd, off, SEEK_SET) == -1) {
-			perror("Lseek on destination failed");
-			write_error = TRUE;
-		}
-
-		if(!write_error && write(fd, file_buffer->data, file_buffer->size) == -1) {
-			perror("Write on destination failed");
-			write_error = TRUE;
-		}
-		pthread_mutex_unlock(&pos_mutex);
-
-		alloc_free(file_buffer);
-	}
-}
-
-
-void *deflator(void *arg)
-{
-	z_stream *stream = NULL;
-	int oldstate;
-
-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
-	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate);
-
-	while(1) {
-		struct file_buffer *file_buffer = queue_get(from_reader);
-		struct file_buffer *write_buffer = alloc_get(writer_buffer);
-
-		write_buffer->c_byte = mangle2(&stream, write_buffer->data, file_buffer->data, file_buffer->size, block_size, noD, 1);
-		write_buffer->block = file_buffer->block;
-		write_buffer->block_order = file_buffer->block_order;
-		write_buffer->size = SQUASHFS_COMPRESSED_SIZE_BLOCK(write_buffer->c_byte);
-		write_buffer->fragment = FALSE;
-		write_buffer->error = FALSE;
-		alloc_free(file_buffer);
-		queue_put(from_deflate, write_buffer);
-	}
-}
-
-
-void *frag_deflator(void *arg)
-{
-	z_stream *stream = NULL;
-	int oldstate;
-
-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
-	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate);
-
-	while(1) {
-		int c_byte, compressed_size;
-		struct file_buffer *file_buffer = queue_get(to_frag);
-		struct file_buffer *write_buffer = alloc_get_2(writer_buffer, frag_release, file_buffer->block);
-
-		c_byte = mangle2(&stream, write_buffer->data, file_buffer->data, file_buffer->size, block_size, noF, 1);
-		compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
-		pthread_mutex_lock(&fragment_mutex);
-		fragment_table[file_buffer->block].size = c_byte;
-		fragment_table[file_buffer->block].start_block = bytes;
-		write_buffer->size = compressed_size;
-		write_buffer->block = bytes;
-		queue_put(to_writer, write_buffer);
-		bytes += compressed_size;
-		total_uncompressed += file_buffer->size;
-		total_compressed += compressed_size;
-		TRACE("Writing fragment %d, uncompressed size %d, compressed size %d\n", file_buffer->block, file_buffer->size, compressed_size);
-		fragments_outstanding --;
-		pthread_cond_signal(&fragment_waiting);
-		pthread_mutex_unlock(&fragment_mutex);
-		alloc_free(file_buffer);
+	if((dupl_ptr = (struct file_info *) malloc(sizeof(struct file_info))) == NULL) {
+		BAD_ERROR("Out of memory in dup_files allocation!\n");
 	}
-}
-
-
-#define HASH_ENTRIES		256
-#define BLOCK_HASH(a)		(a % HASH_ENTRIES)
-struct file_buffer		*block_hash[HASH_ENTRIES];
 
-void push_buffer(struct file_buffer *file_buffer)
-{
-	int hash = BLOCK_HASH(file_buffer->block_order);
-
-	file_buffer->next = block_hash[hash];
-	block_hash[hash] = file_buffer;
-}
-
-
-struct file_buffer *get_file_buffer(struct queue *queue)
-{
-	static unsigned int block_order = 0;
-	int hash = BLOCK_HASH(block_order);
-	struct file_buffer *file_buffer = block_hash[hash], *prev = NULL;
-
-	for(;file_buffer; prev = file_buffer, file_buffer = file_buffer->next)
-		if(file_buffer->block_order == block_order)
-			break;
+	dupl_ptr->bytes = bytes;
+	dupl_ptr->checksum = checksum;
+	dupl_ptr->start = *start;
+	dupl_ptr->fragment_checksum = fragment_checksum;
+	dupl_ptr->block_list = *block_list;
 
-	if(file_buffer) {
-		if(prev)
-			prev->next = file_buffer->next;
-		else
-			block_hash[hash] = file_buffer->next;
+	dup_files ++;
+	if(bytes) {
+		dupl_ptr->next = dupl[checksum];
+		dupl[checksum] = dupl_ptr;
 	} else {
-		while(1) {
-			file_buffer = queue_get(queue);
-			if(file_buffer->block_order == block_order)
-				break;
-			push_buffer(file_buffer);
-		}
+		dupl_ptr->next = frag_dups[fragment_checksum];
+		frag_dups[fragment_checksum] = dupl_ptr;
 	}
 
-	block_order ++;
-
-	return file_buffer;
-}
-
-
-int write_file_empty(squashfs_inode *inode, struct dir_ent *dir_ent, int *duplicate_file)
-{
-	file_count ++;
-	*duplicate_file = FALSE;
-	return dir_ent->inode->nlink == 1 ?
-		create_inode(inode, dir_ent, SQUASHFS_FILE_TYPE, 0, 0, 0, NULL, &empty_fragment, NULL) :
-		create_inode(inode, dir_ent, SQUASHFS_LREG_TYPE, 0, 0, 0, NULL, &empty_fragment, NULL);
-}
-
-
-int write_file_frag_dup(squashfs_inode *inode, struct dir_ent *dir_ent, int size, int *duplicate_file, struct file_buffer *file_buffer, unsigned short checksum)
-{
-	int file;
-	struct file_info *dupl_ptr;
-	struct fragment *fragment;
-	unsigned int *block_listp = NULL;
-	long long start = 0;
-
-	dupl_ptr = duplicate(size, 0, &block_listp, &start, &fragment, file_buffer, NULL, 0, 0, checksum, TRUE);
-
-	if(dupl_ptr) {
-		*duplicate_file = FALSE;
-		fragment = get_and_fill_fragment(file_buffer);
-		dupl_ptr->fragment = fragment;
-	} else
-		*duplicate_file = TRUE;
-
-	alloc_free(file_buffer);
-
-	total_bytes += size;
-	file_count ++;
-
-	return dir_ent->inode->nlink == 1 ?
-		create_inode(inode, dir_ent, SQUASHFS_FILE_TYPE, size, 0, 0, NULL, fragment, NULL) :
-		create_inode(inode, dir_ent, SQUASHFS_LREG_TYPE, size, 0, 0, NULL, fragment, NULL);
-}
-
-
-int write_file_frag(squashfs_inode *inode, struct dir_ent *dir_ent, int size, int *duplicate_file)
-{
-	struct fragment *fragment;
-	unsigned short checksum;
-	struct file_buffer *file_buffer = get_file_buffer(from_deflate);
-
-	if(file_buffer->size != size)
-		printf("bug\n");
-
-	if(file_buffer->error) {
-		alloc_free(file_buffer);
-		return FALSE;
-	}
-
-	checksum = get_checksum_mem_buffer(file_buffer);
-
-	if(pre_duplicate_frag(size, checksum))
-		return write_file_frag_dup(inode, dir_ent, size, duplicate_file, file_buffer, checksum);
-		
-	fragment = get_and_fill_fragment(file_buffer);
-
-	alloc_free(file_buffer);
-
-	if(duplicate_checking)
-		add_non_dup(size, 0, NULL, 0, fragment, 0, checksum, TRUE);
-
-	total_bytes += size;
-	file_count ++;
-
-	*duplicate_file = FALSE;
-
-	return dir_ent->inode->nlink == 1 ?
-		create_inode(inode, dir_ent, SQUASHFS_FILE_TYPE, size, 0, 0, NULL, fragment, NULL) :
-		create_inode(inode, dir_ent, SQUASHFS_LREG_TYPE, size, 0, 0, NULL, fragment, NULL);
+	return dupl_ptr;
 }
 
 
-int write_file_blocks(squashfs_inode *inode, struct dir_ent *dir_ent, long long read_size)
+#define MINALLOCBYTES (1024 * 1024)
+int write_file(squashfs_inode *inode, struct dir_ent *dir_ent, long long size, int *duplicate_file)
 {
-	int block, status;
+	int block = 0, i, file, whole_file = 1, status;
 	unsigned int c_byte, frag_bytes;
-	long long bbytes, file_bytes, start;
+	long long bbytes, file_bytes = 0, start;
+	char buff[block_size], *c_buffer = NULL, *filename = dir_ent->pathname;
 	struct fragment *fragment;
-	struct file_info *dupl_ptr;
-	int blocks = (read_size + block_size - 1) >> block_log;
-	unsigned int *block_list;
-	struct file_buffer *read_buffer;
-
-	if(!no_fragments && always_use_fragments) {
-		blocks = read_size >> block_log;
-		frag_bytes = read_size % block_size;
-	} else
-		frag_bytes = 0;
+	struct file_info *dupl_ptr = NULL;
+	struct duplicate_buffer_handle handle;
+	long long read_size = (size > SQUASHFS_MAX_FILE_SIZE) ? SQUASHFS_MAX_FILE_SIZE : size;
+	int blocks = (read_size + block_size - 1) >> block_log, allocated_blocks = blocks;
+	unsigned int *block_list, *block_listp;
 
 	if((block_list = malloc(blocks * sizeof(unsigned int))) == NULL)
 		BAD_ERROR("Out of memory allocating block_list\n");
+	block_listp = block_list;
 
-	ensure_fragments_flushed();
-
-	file_bytes = 0;
-	start = bytes;
-	for(block = 0; block < blocks; block ++) {
-		read_buffer = get_file_buffer(from_deflate);
-		if(read_buffer->error)
-			goto read_err;
-
-		block_list[block] = read_buffer->c_byte;
-		read_buffer->block = bytes;
-		bytes += read_buffer->size;
-		file_bytes += read_buffer->size;
-		queue_put(to_writer, read_buffer);
-	}
-
-	if(frag_bytes != 0) {
-		read_buffer = get_file_buffer(from_deflate);
-		if(read_buffer->size != frag_bytes)
-			printf("bug\n");
-		if(read_buffer->error)
-			goto read_err;
-	} else
-		read_buffer = NULL;
-
-	fragment = get_and_fill_fragment(read_buffer);
-	alloc_free(read_buffer);
-
-	if(duplicate_checking)
-		add_non_dup(read_size, file_bytes, block_list, start, fragment, 0, 0, FALSE);
-	file_count ++;
-	total_bytes += read_size;
-	if(dir_ent->inode->nlink == 1 && read_size < ((long long) (1<<30) - 1))
-		status = create_inode(inode, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, blocks, block_list, fragment, NULL);
-	else
-		status = create_inode(inode, dir_ent, SQUASHFS_LREG_TYPE, read_size, start, blocks, block_list, fragment, NULL);
-	if(duplicate_checking == FALSE)
-		free(block_list);
-	return status;
-
-read_err:
-	perror("Error in reading file, skipping...");
-	if(block) {
-		queue_put(to_writer, NULL);
-		if(queue_get(from_writer) != 0)
-			EXIT_MKSQUASHFS();
-		bytes = start;
-		if(!block_device)
-			ftruncate(fd, bytes);
-	}
-	free(block_list);
-	alloc_free(read_buffer);
-	return FALSE;
-}
-
-
-int write_file_blocks_dup(squashfs_inode *inode, struct dir_ent *dir_ent, long long read_size, int *duplicate_file)
-{
-	int block, status, thresh;
-	unsigned int c_byte, frag_bytes;
-	long long bbytes, file_bytes, start;
-	struct fragment *fragment;
-	struct file_info *dupl_ptr;
-	int blocks = (read_size + block_size - 1) >> block_log;
-	unsigned int *block_list, *block_listp;
-	struct file_buffer *read_buffer;
-	struct file_data *file_data;
-	struct buffer_list *buffer_list;
-
-	if(!no_fragments && always_use_fragments) {
-		blocks = read_size >> block_log;
+	if(!no_fragments && (read_size < block_size || always_use_fragments)) {
+		allocated_blocks = blocks = read_size >> block_log;
 		frag_bytes = read_size % block_size;
 	} else
 		frag_bytes = 0;
 
-	if((block_list = malloc(blocks * sizeof(unsigned int))) == NULL)
-		BAD_ERROR("Out of memory allocating block_list\n");
-	block_listp = block_list;
-
-	if((buffer_list = malloc(blocks * sizeof(struct buffer_list))) == NULL)
-		BAD_ERROR("Out of memory allocating file block list\n");
-
-	ensure_fragments_flushed();
+	if(size > read_size)
+		ERROR("file %s truncated to %lld bytes\n", filename, SQUASHFS_MAX_FILE_SIZE);
 
-	file_bytes = 0;
-	start = bytes;
-	thresh = blocks > (writer_buffer_size - processors) ? blocks - (writer_buffer_size - processors): 0;
-	for(block = 0; block < blocks; block ++) {
-		read_buffer = get_file_buffer(from_deflate);
-		if(read_buffer->error)
-			goto read_err;
+	total_bytes += read_size;
+	if((file = open(filename, O_RDONLY)) == -1)
+		goto read_err;
 
-		block_list[block] = read_buffer->c_byte;
-		read_buffer->block = bytes;
-		bytes += read_buffer->size;
-		file_bytes += read_buffer->size;
-
-		if(block < thresh) {
-			buffer_list[block].read_buffer = NULL;
-			queue_put(to_writer, read_buffer);
-		} else
-			buffer_list[block].read_buffer = read_buffer;
-		buffer_list[block].start = read_buffer->block;
-		buffer_list[block].size = read_buffer->size;
+	do {
+		long long bytes = (((long long) allocated_blocks) + 1) << block_log;
+		if(bytes != ((size_t) bytes) || (c_buffer = (char *) malloc(bytes)) == NULL) {
+			TRACE("Out of memory allocating write_file buffer, allocated_blocks %ld, blocks %d\n", allocated_blocks, blocks);
+			whole_file = 0;
+			if(bytes < MINALLOCBYTES)
+				BAD_ERROR("Out of memory allocating write_file buffer, could not allocate %ld blocks (%d Kbytes)\n", allocated_blocks, allocated_blocks << (block_log - 10));
+			allocated_blocks >>= 1;
+		}
+	} while(!c_buffer);
+
+	for(start = bytes; block < blocks; file_bytes += bbytes) {
+		for(i = 0, bbytes = 0; (i < allocated_blocks) && (block < blocks); i++) {
+			int available_bytes = read_size - (block * block_size) > block_size ? block_size : read_size - (block * block_size);
+			if(read(file, buff, available_bytes) == -1)
+				goto read_err;
+			c_byte = mangle(c_buffer + bbytes, buff, available_bytes, block_size, noD, 1);
+			block_list[block ++] = c_byte;
+			bbytes += SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
+		}
+		if(!whole_file) {
+			write_bytes(fd, bytes, bbytes, c_buffer);
+			bytes += bbytes;
+		}
 	}
 
-	if(frag_bytes != 0) {
-		read_buffer = get_file_buffer(from_deflate);
-		if(read_buffer->size != frag_bytes)
-			printf("bug\n");
-		if(read_buffer->error)
+	if(frag_bytes != 0)
+		if(read(file, buff, frag_bytes) == -1)
 			goto read_err;
-	} else
-		read_buffer = NULL;
-
-	queue_put(to_writer, NULL);
-	if(queue_get(from_writer) != 0)
-		EXIT_MKSQUASHFS();
-
-	dupl_ptr = duplicate(read_size, file_bytes, &block_listp, &start, &fragment, read_buffer, buffer_list, blocks, 0, 0, FALSE);
 
-	if(dupl_ptr) {
-		*duplicate_file = FALSE;
-		for(block = thresh; block < blocks; block ++)
-			queue_put(to_writer, buffer_list[block].read_buffer);
-		fragment = get_and_fill_fragment(read_buffer);
-		dupl_ptr->fragment = fragment;
+	close(file);
+	if(whole_file) {
+		handle.ptr = c_buffer;
+		if(duplicate_checking && (dupl_ptr = duplicate(read_from_buffer, &handle, file_bytes, &block_listp, &start, blocks, &fragment, buff, frag_bytes)) == NULL) {
+			*duplicate_file = TRUE;
+			goto wr_inode;
+		}
+		write_bytes(fd, bytes, file_bytes, c_buffer);
+		bytes += file_bytes;
 	} else {
-		*duplicate_file = TRUE;
-		for(block = thresh; block < blocks; block ++)
-			alloc_free(buffer_list[block].read_buffer);
-		bytes = buffer_list[0].start;
-		if(thresh && !block_device)
-			ftruncate(fd, bytes);
+		handle.start = start;
+		if(duplicate_checking && (dupl_ptr = duplicate(read_from_file, &handle, file_bytes, &block_listp, &start, blocks, &fragment, buff, frag_bytes)) == NULL) {
+			bytes = start;
+			if(!block_device)
+				ftruncate(fd, bytes);
+			*duplicate_file = TRUE;
+			goto wr_inode;
+		}
 	}
 
-	alloc_free(read_buffer);
-	free(buffer_list);
-	file_count ++;
-	total_bytes += read_size;
+	fragment = get_and_fill_fragment(buff, frag_bytes);
+	if(duplicate_checking)
+		dupl_ptr->fragment = fragment;
+
+	*duplicate_file = FALSE;
 
+wr_inode:
+	free(c_buffer);
+	file_count ++;
 	if(dir_ent->inode->nlink == 1 && read_size < ((long long) (1<<30) - 1))
 		status = create_inode(inode, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, blocks, block_listp, fragment, NULL);
 	else
 		status = create_inode(inode, dir_ent, SQUASHFS_LREG_TYPE, read_size, start, blocks, block_listp, fragment, NULL);
-	if(*duplicate_file == TRUE)
+	if(duplicate_checking == FALSE || *duplicate_file == TRUE)
 		free(block_list);
 	return status;
 
 read_err:
 	perror("Error in reading file, skipping...");
-	if(block && thresh) {
-		queue_put(to_writer, NULL);
-		if(queue_get(from_writer) != 0)
-			EXIT_MKSQUASHFS();
-		bytes = start;
-		if(!block_device)
-			ftruncate(fd, bytes);
-	}
-	for(blocks = thresh; blocks < block; blocks ++)
-		alloc_free(buffer_list[blocks].read_buffer);
-	free(buffer_list);
+	free(c_buffer);
 	free(block_list);
-	alloc_free(read_buffer);
 	return FALSE;
 }
 
 
-int write_file(squashfs_inode *inode, struct dir_ent *dir_ent, long long size, int *duplicate_file)
-{
-	long long read_size = (size > SQUASHFS_MAX_FILE_SIZE) ? SQUASHFS_MAX_FILE_SIZE : size;
-
-	if(size > read_size)
-		ERROR("file %s truncated to %lld bytes\n", dir_ent->pathname, SQUASHFS_MAX_FILE_SIZE);
-
-	if(read_size == 0)
-		return write_file_empty(inode, dir_ent, duplicate_file);
-
-	if(!no_fragments && (read_size < block_size))
-		return write_file_frag(inode, dir_ent, read_size, duplicate_file);
-
-	if(pre_duplicate(read_size))
-		return write_file_blocks_dup(inode, dir_ent, read_size, duplicate_file);
-
-	*duplicate_file = FALSE;
-	return write_file_blocks(inode, dir_ent, read_size);
-}
-
-
 char b_buffer[8192];
 char *name;
 char *basename_r();
@@ -2134,7 +1334,6 @@
 		BAD_ERROR("Out of memory in inode hash table entry allocation\n");
 
 	memcpy(&inode->buf, buf, sizeof(struct stat));
-	inode->read = FALSE;
 	inode->inode = SQUASHFS_INVALID_BLK;
 	inode->nlink = 1;
 	if((buf->st_mode & S_IFMT) == S_IFDIR)
@@ -2357,9 +1556,6 @@
 		return;
 	}
 	if(sorted)
-		generate_file_priorities(dir_info, 0, &dir_info->dir_ent->inode->buf);
-	queue_put(to_reader, dir_info);
-	if(sorted)
 		sort_files_and_write(dir_info);
 	dir_scan2(inode, dir_info);
 }
@@ -2495,7 +1691,7 @@
 				case SQUASHFS_CHRDEV_TYPE:
 					INFO("character device %s inode 0x%llx LINK\n", dir_name, *inode);
 					break;
-				case SQUASHFS_BLKDEV_TYPE:
+				caseSQUASHFS_BLKDEV_TYPE:
 					INFO("block device %s inode 0x%llx LINK\n", dir_name, *inode);
 					break;
 				case SQUASHFS_FIFO_TYPE:
@@ -2595,76 +1791,8 @@
 }
 
 
-void initialise_threads()
-{
-	int i;
-	sigset_t sigmask, old_mask;
-
-	sigemptyset(&sigmask);
-	sigaddset(&sigmask, SIGINT);
-	sigaddset(&sigmask, SIGQUIT);
-	if(sigprocmask(SIG_BLOCK, &sigmask, &old_mask) == -1)
-		BAD_ERROR("Failed to set signal mask in intialise_threads\n");
-
-	signal(SIGUSR1, sigusr1_handler);
-
-	if(processors == -1) {
-#ifndef linux
-		int mib[2];
-		size_t len = sizeof(processors);
-
-		mib[0] = CTL_HW;
-#ifdef HW_AVAILCPU
-		mib[1] = HW_AVAILCPU;
-#else
-		mib[1] = HW_NCPU;
-#endif
-
-		if(sysctl(mib, 2, &processors, &len, NULL, 0) == -1) {
-			ERROR("Failed to get number of available processors.  Defaulting to 1\n");
-			processors = 1;
-		}
-#else
-		processors = get_nprocs();
-#endif
-	}
-
-	if((thread = malloc((2 + processors * 2) * sizeof(pthread_t))) == NULL)
-		BAD_ERROR("Out of memory allocating thread descriptors\n");
-	deflator_thread = &thread[2];
-	frag_deflator_thread = &deflator_thread[processors];
-
-	to_reader = queue_init(1);
-	from_reader = queue_init(reader_buffer_size);
-	to_writer = queue_init(writer_buffer_size);
-	from_writer = queue_init(1);
-	from_deflate = queue_init(reader_buffer_size);
-	to_frag = queue_init(processors * 2);
-	reader_buffer = alloc_init(SQUASHFS_FILE_MAX_SIZE, reader_buffer_size);
-	writer_buffer = alloc_init(SQUASHFS_FILE_MAX_SIZE, writer_buffer_size);
-	fragment_buffer = alloc_init(SQUASHFS_FILE_MAX_SIZE, processors * 2);
-	pthread_create(&thread[0], NULL, reader, NULL);
-	pthread_create(&thread[1], NULL, writer, NULL);
-	pthread_mutex_init(&fragment_mutex, NULL);
-	pthread_cond_init(&fragment_waiting, NULL);
-
-	for(i = 0; i < processors; i++) {
-		if(pthread_create(&deflator_thread[i], NULL, deflator, NULL) != 0 )
-			BAD_ERROR("Failed to create thread\n");
-		if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator, NULL) != 0)
-			BAD_ERROR("Failed to create thread\n");
-	}
-
-	printf("Parallel mksquashfs: Using %d processor%s\n", processors,
-			processors == 1 ? "" : "s");
-
-	if(sigprocmask(SIG_SETMASK, &old_mask, NULL) == -1)
-		BAD_ERROR("Failed to set signal mask in intialise_threads\n");
-}
-
-
 #define VERSION() \
-	printf("mksquashfs version 3.1-r2 (2006/08/30)\n");\
+	printf("mksquashfs version 3.0 (2006/03/15)\n");\
 	printf("copyright (C) 2006 Phillip Lougher <phillip@lougher.org.uk>\n\n"); \
     	printf("This program is free software; you can redistribute it and/or\n");\
 	printf("modify it under the terms of the GNU General Public License\n");\
@@ -2682,7 +1810,6 @@
 	char *b, *root_name = NULL;
 	int be, nopad = FALSE, keep_as_directory = FALSE, orig_be;
 	squashfs_inode inode;
-	int readb_mbytes = READER_BUFFER_DEFAULT, writeb_mbytes = WRITER_BUFFER_DEFAULT;
 
 #if __BYTE_ORDER == __BIG_ENDIAN
 	be = TRUE;
@@ -2701,34 +1828,7 @@
 	source_path = argv + 1;
 	source = i - 2;
 	for(; i < argc; i++) {
-		if(strcmp(argv[i], "-processors") == 0) {
-			if((++i == argc) || (processors = strtol(argv[i], &b, 10), *b != '\0')) {
-				ERROR("%s: -processors missing or invalid processor number\n", argv[0]);
-				exit(1);
-			}
-			if(processors < 1) {
-				ERROR("%s: -processors should be 1 or larger\n", argv[0]);
-				exit(1);
-			}
-		} else if(strcmp(argv[i], "-read_queue") == 0) {
-			if((++i == argc) || (readb_mbytes = strtol(argv[i], &b, 10), *b != '\0')) {
-				ERROR("%s: -read_queue missing or invalid queue size\n", argv[0]);
-				exit(1);
-			}
-			if(readb_mbytes < 1) {
-				ERROR("%s: -read_queue should be 1 megabyte or larger\n", argv[0]);
-				exit(1);
-			}
-		} else if(strcmp(argv[i], "-write_queue") == 0) {
-			if((++i == argc) || (writeb_mbytes = strtol(argv[i], &b, 10), *b != '\0')) {
-				ERROR("%s: -write_queue missing or invalid queue size\n", argv[0]);
-				exit(1);
-			}
-			if(writeb_mbytes < 1) {
-				ERROR("%s: -write_queue should be 1 megabyte or larger\n", argv[0]);
-				exit(1);
-			}
-		} else if(strcmp(argv[i], "-b") == 0) {
+		if(strcmp(argv[i], "-b") == 0) {
 			if((++i == argc) || (block_size = strtol(argv[i], &b, 10), *b !='\0')) {
 				ERROR("%s: -b missing or invalid block size\n", argv[0]);
 				exit(1);
@@ -2851,9 +1951,6 @@
 			ERROR("-version\t\tprint version, licence and copyright message\n");
 			ERROR("-info\t\t\tprint files written to filesystem\n");
 			ERROR("-b <block_size>\t\tset data block to <block_size>.  Default %d bytes\n", SQUASHFS_FILE_SIZE);
-			ERROR("-processors <number>\tUse <number> processors.  By default will use number of\n\t\t\tprocessors available\n");
-			ERROR("-read-queue <size>\tSet input queue to <size> Mbytes.  Default %d Mbytes\n", READER_BUFFER_DEFAULT);
-			ERROR("-write-queue <size>\tSet output queue to <size> Mbytes.  Default %d Mbytes\n", WRITER_BUFFER_DEFAULT);
 			ERROR("-noI\t\t\tdo not compress inode table\n");
 			ERROR("-noD\t\t\tdo not compress data blocks\n");
 			ERROR("-noF\t\t\tdo not compress fragment blocks\n");
@@ -2887,9 +1984,6 @@
 		}
 	}
 
-	reader_buffer_size = readb_mbytes << (20 - block_log);
-	writer_buffer_size = writeb_mbytes << (20 - block_log);
-
         for(i = 0; i < source; i++)
                 if(stat(source_path[i], &source_buf) == -1) {
                         fprintf(stderr, "Cannot stat source directory \"%s\" because %s\n", source_path[i], strerror(errno));
@@ -2975,8 +2069,6 @@
 		else if(strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "-root-becomes") == 0 || strcmp(argv[i], "-ef") == 0)
 			i++;
 
-	initialise_threads();
-
 	if(delete) {
 		printf("Creating %s %d.%d filesystem on %s, block size %d.\n",
 				be ? "big endian" : "little endian", SQUASHFS_MAJOR, SQUASHFS_MINOR, argv[source + 1], block_size);
@@ -3104,13 +2196,6 @@
 restore_filesystem:
 	write_fragment();
 	sBlk.fragments = fragments;
-	if(interrupted < 2) {
-		ensure_fragments_flushed();
-		queue_put(to_writer, NULL);
-		if(queue_get(from_writer) != 0)
-			EXIT_MKSQUASHFS();
-	}
-
 	sBlk.inode_table_start = write_inodes();
 	sBlk.directory_table_start = write_directories();
 	sBlk.fragment_table_start = write_fragment_table();
diff -Nur squashfs-tools/read_fs.c squashfs-lzma-tools/read_fs.c
--- squashfs-tools/read_fs.c	2006-08-21 01:40:35.000000000 +0200
+++ squashfs-lzma-tools/read_fs.c	2006-08-22 09:39:31.000000000 +0200
@@ -22,7 +22,7 @@
  */
 
 extern void read_bytes(int, long long, int, char *);
-extern int add_file(long long, long long, long long, unsigned int *, int, unsigned int, int, int);
+extern int add_file(long long, long long, unsigned int *, int, unsigned int, int, int);
 
 #define TRUE 1
 #define FALSE 0
@@ -186,8 +186,7 @@
 					+ sBlk->block_size - 1) >> sBlk->block_log : inode.file_size >>
 					sBlk->block_log;
 				long long file_bytes = 0;
-				int i;
-				long long start = inode.start_block;
+				int i, start = inode.start_block;
 				unsigned int *block_list;
 
 				TRACE("scan_inode_table: regular file, file_size %lld, blocks %d\n", inode.file_size, blocks);
@@ -211,7 +210,7 @@
 				for(i = 0; i < blocks; i++)
 					file_bytes += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]);
 
-	                        add_file(start, inode.file_size, file_bytes, block_list, blocks, inode.fragment, inode.offset, frag_bytes);
+	                        add_file(start, file_bytes, block_list, blocks, inode.fragment, inode.offset, frag_bytes);
 				cur_ptr += blocks * sizeof(unsigned int);
 				break;
 			}	
@@ -220,8 +219,7 @@
 				int frag_bytes;
 				int blocks;
 				long long file_bytes = 0;
-				int i;
-				long long start;
+				int i, start;
 				unsigned int *block_list;
 
 				if(swap) {
@@ -258,7 +256,7 @@
 				for(i = 0; i < blocks; i++)
 					file_bytes += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]);
 
-	                        add_file(start, inode.file_size, file_bytes, block_list, blocks, inode.fragment, inode.offset, frag_bytes);
+	                        add_file(start, file_bytes, block_list, blocks, inode.fragment, inode.offset, frag_bytes);
 				cur_ptr += blocks * sizeof(unsigned int);
 				break;
 			}	
diff -Nur squashfs-tools/README squashfs-lzma-tools/README
--- squashfs-tools/README	1970-01-01 01:00:00.000000000 +0100
+++ squashfs-lzma-tools/README	2006-08-22 09:36:18.000000000 +0200
@@ -0,0 +1,2 @@
+This is mksquashfs patched with LZMA support
+and with the patch for 'fragment_table rounding bug'
\ No newline at end of file
diff -Nur squashfs-tools/sort.c squashfs-lzma-tools/sort.c
--- squashfs-tools/sort.c	2006-08-21 01:40:35.000000000 +0200
+++ squashfs-lzma-tools/sort.c	2006-06-25 05:17:43.000000000 +0200
@@ -71,6 +71,11 @@
 
 struct sort_info *sort_info_list[65536];
 
+struct priority_entry {
+	struct dir_ent *dir;
+	struct priority_entry *next;
+};
+
 struct priority_entry *priority_list[65536];
 
 extern int silent;
@@ -230,6 +235,8 @@
 	squashfs_inode inode;
 	int duplicate_file;
 
+	generate_file_priorities(dir, 0, &dir->dir_ent->inode->buf);
+
 	for(i = 65535; i >= 0; i--)
 		for(entry = priority_list[i]; entry; entry = entry->next) {
 			TRACE("%d: %s\n", i - 32768, entry->dir->pathname);
diff -Nur squashfs-tools/sort.h squashfs-lzma-tools/sort.h
--- squashfs-tools/sort.h	2006-08-21 01:40:35.000000000 +0200
+++ squashfs-lzma-tools/sort.h	2006-06-25 05:17:43.000000000 +0200
@@ -51,12 +51,6 @@
 	squashfs_inode		inode;
 	unsigned int		type;
 	unsigned int		inode_number;
-	char			read;
 	struct inode_info	*next;
 };
-
-struct priority_entry {
-	struct dir_ent *dir;
-	struct priority_entry *next;
-};
 #endif
diff -Nur squashfs-tools/squashfs_fs.h squashfs-lzma-tools/squashfs_fs.h
--- squashfs-tools/squashfs_fs.h	2006-08-21 02:00:22.000000000 +0200
+++ squashfs-lzma-tools/squashfs_fs.h	2006-06-25 05:17:43.000000000 +0200
@@ -308,7 +308,7 @@
 struct squashfs_fragment_entry {
 	long long		start_block;
 	unsigned int		size;
-	unsigned int		pending;
+	unsigned int		unused;
 } __attribute__ ((packed));
 
 extern int squashfs_uncompress_block(void *d, int dstlen, void *s, int srclen);