backsql_srch_query( backsql_srch_info *bsi, struct berval *query )
{
	backsql_info		*bi = (backsql_info *)bsi->bsi_op->o_bd->be_private;
	int			rc;

	assert( query != NULL );
	BER_BVZERO( query );

	bsi->bsi_use_subtree_shortcut = 0;

	Debug( LDAP_DEBUG_TRACE, "==>backsql_srch_query()\n" );
	BER_BVZERO( &bsi->bsi_sel.bb_val );
	BER_BVZERO( &bsi->bsi_sel.bb_val );
	bsi->bsi_sel.bb_len = 0;
	BER_BVZERO( &bsi->bsi_from.bb_val );
	bsi->bsi_from.bb_len = 0;
	BER_BVZERO( &bsi->bsi_join_where.bb_val );
	bsi->bsi_join_where.bb_len = 0;
	BER_BVZERO( &bsi->bsi_flt_where.bb_val );
	bsi->bsi_flt_where.bb_len = 0;

	backsql_strfcat_x( &bsi->bsi_sel,
			bsi->bsi_op->o_tmpmemctx,
			"lbcbc",
			(ber_len_t)STRLENOF( "SELECT DISTINCT ldap_entries.id," ),
				"SELECT DISTINCT ldap_entries.id,",
			&bsi->bsi_oc->bom_keytbl,
			'.',
			&bsi->bsi_oc->bom_keycol,
			',' );

	if ( !BER_BVISNULL( &bi->sql_strcast_func ) ) {
		backsql_strfcat_x( &bsi->bsi_sel,
				bsi->bsi_op->o_tmpmemctx,
				"blbl",
				&bi->sql_strcast_func,
				(ber_len_t)STRLENOF( "('" /* ') */ ),
					"('" /* ') */ ,
				&bsi->bsi_oc->bom_oc->soc_cname,
				(ber_len_t)STRLENOF( /* (' */ "')" ),
					/* (' */ "')" );
	} else {
		backsql_strfcat_x( &bsi->bsi_sel,
				bsi->bsi_op->o_tmpmemctx,
				"cbc",
				'\'',
				&bsi->bsi_oc->bom_oc->soc_cname,
				'\'' );
	}

	backsql_strfcat_x( &bsi->bsi_sel,
			bsi->bsi_op->o_tmpmemctx,
			"b",
			&bi->sql_dn_oc_aliasing );
	backsql_strfcat_x( &bsi->bsi_from,
			bsi->bsi_op->o_tmpmemctx,
			"lb",
			(ber_len_t)STRLENOF( " FROM ldap_entries," ),
				" FROM ldap_entries,",
			&bsi->bsi_oc->bom_keytbl );

	backsql_strfcat_x( &bsi->bsi_join_where,
			bsi->bsi_op->o_tmpmemctx,
			"lbcbl",
			(ber_len_t)STRLENOF( " WHERE " ), " WHERE ",
			&bsi->bsi_oc->bom_keytbl,
			'.',
			&bsi->bsi_oc->bom_keycol,
			(ber_len_t)STRLENOF( "=ldap_entries.keyval AND ldap_entries.oc_map_id=? AND " ),
				"=ldap_entries.keyval AND ldap_entries.oc_map_id=? AND " );

	switch ( bsi->bsi_scope ) {
	case LDAP_SCOPE_BASE:
		if ( BACKSQL_CANUPPERCASE( bi ) ) {
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"bl",
					&bi->sql_upper_func,
					(ber_len_t)STRLENOF( "(ldap_entries.dn)=?" ),
						"(ldap_entries.dn)=?" );
		} else {
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"l",
					(ber_len_t)STRLENOF( "ldap_entries.dn=?" ),
						"ldap_entries.dn=?" );
		}
		break;

	case BACKSQL_SCOPE_BASE_LIKE:
		if ( BACKSQL_CANUPPERCASE( bi ) ) {
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"bl",
					&bi->sql_upper_func,
					(ber_len_t)STRLENOF( "(ldap_entries.dn) LIKE ?" ),
						"(ldap_entries.dn) LIKE ?" );
		} else {
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"l",
					(ber_len_t)STRLENOF( "ldap_entries.dn LIKE ?" ),
						"ldap_entries.dn LIKE ?" );
		}
		break;

	case LDAP_SCOPE_ONELEVEL:
		backsql_strfcat_x( &bsi->bsi_join_where,
				bsi->bsi_op->o_tmpmemctx,
				"l",
				(ber_len_t)STRLENOF( "ldap_entries.parent=?" ),
					"ldap_entries.parent=?" );
		break;

	case LDAP_SCOPE_SUBORDINATE:
	case LDAP_SCOPE_SUBTREE:
		if ( BACKSQL_USE_SUBTREE_SHORTCUT( bi ) ) {
			int		i;
			BackendDB	*bd = bsi->bsi_op->o_bd;

			assert( bd->be_nsuffix != NULL );

			for ( i = 0; !BER_BVISNULL( &bd->be_nsuffix[ i ] ); i++ )
			{
				if ( dn_match( &bd->be_nsuffix[ i ],
							bsi->bsi_base_ndn ) )
				{
					/* pass this to the candidate selection
					 * routine so that the DN is not bound
					 * to the select statement */
					bsi->bsi_use_subtree_shortcut = 1;
					break;
				}
			}
		}

		if ( bsi->bsi_use_subtree_shortcut ) {
			/* Skip the base DN filter, as every entry will match it */
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"l",
					(ber_len_t)STRLENOF( "9=9"), "9=9");

		} else if ( !BER_BVISNULL( &bi->sql_subtree_cond ) ) {
			/* This should always be true... */
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"b",
					&bi->sql_subtree_cond );

		} else if ( BACKSQL_CANUPPERCASE( bi ) ) {
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"bl",
					&bi->sql_upper_func,
					(ber_len_t)STRLENOF( "(ldap_entries.dn) LIKE ?" ),
						"(ldap_entries.dn) LIKE ?"  );

		} else {
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"l",
					(ber_len_t)STRLENOF( "ldap_entries.dn LIKE ?" ),
						"ldap_entries.dn LIKE ?" );
		}

		break;

	default:
		assert( 0 );
	}

#ifndef BACKSQL_ARBITRARY_KEY
	/* If paged results are in effect, ignore low ldap_entries.id numbers */
	if ( get_pagedresults(bsi->bsi_op) > SLAP_CONTROL_IGNORED ) {
		unsigned long lowid = 0;

		/* Pick up the previous ldap_entries.id if the previous page ended in this objectClass */
		if ( bsi->bsi_oc->bom_id == PAGECOOKIE_TO_SQL_OC( ((PagedResultsState *)bsi->bsi_op->o_pagedresults_state)->ps_cookie ) )
		{
			lowid = PAGECOOKIE_TO_SQL_ID( ((PagedResultsState *)bsi->bsi_op->o_pagedresults_state)->ps_cookie );
		}

		if ( lowid ) {
			char lowidstring[48];
			int  lowidlen;

			lowidlen = snprintf( lowidstring, sizeof( lowidstring ),
				" AND ldap_entries.id>%lu", lowid );
			backsql_strfcat_x( &bsi->bsi_join_where,
					bsi->bsi_op->o_tmpmemctx,
					"l",
					(ber_len_t)lowidlen,
					lowidstring );
		}
	}
#endif /* ! BACKSQL_ARBITRARY_KEY */

	rc = backsql_process_filter( bsi, bsi->bsi_filter );
	if ( rc > 0 ) {
		struct berbuf	bb = BB_NULL;

		backsql_strfcat_x( &bb,
				bsi->bsi_op->o_tmpmemctx,
				"bbblb",
				&bsi->bsi_sel.bb_val,
				&bsi->bsi_from.bb_val,
				&bsi->bsi_join_where.bb_val,
				(ber_len_t)STRLENOF( " AND " ), " AND ",
				&bsi->bsi_flt_where.bb_val );

		*query = bb.bb_val;

	} else if ( rc < 0 ) {
		/*
		 * Indicates that there's no possible way the filter matches
		 * anything.  No need to issue the query
		 */
		free( query->bv_val );
		BER_BVZERO( query );
	}

	bsi->bsi_op->o_tmpfree( bsi->bsi_sel.bb_val.bv_val, bsi->bsi_op->o_tmpmemctx );
	BER_BVZERO( &bsi->bsi_sel.bb_val );
	bsi->bsi_sel.bb_len = 0;
	bsi->bsi_op->o_tmpfree( bsi->bsi_from.bb_val.bv_val, bsi->bsi_op->o_tmpmemctx );
	BER_BVZERO( &bsi->bsi_from.bb_val );
	bsi->bsi_from.bb_len = 0;
	bsi->bsi_op->o_tmpfree( bsi->bsi_join_where.bb_val.bv_val, bsi->bsi_op->o_tmpmemctx );
	BER_BVZERO( &bsi->bsi_join_where.bb_val );
	bsi->bsi_join_where.bb_len = 0;
	bsi->bsi_op->o_tmpfree( bsi->bsi_flt_where.bb_val.bv_val, bsi->bsi_op->o_tmpmemctx );
	BER_BVZERO( &bsi->bsi_flt_where.bb_val );
	bsi->bsi_flt_where.bb_len = 0;

	Debug( LDAP_DEBUG_TRACE, "<==backsql_srch_query() returns %s\n",
		query->bv_val ? query->bv_val : "NULL" );

	return ( rc <= 0 ? 1 : 0 );
}


static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
	set_64bit((unsigned long long *)(ptep), native_pte_val(pte));
}


EXPORTED int mailbox_get_annotate_state(struct mailbox *mailbox,
                               unsigned int uid,
                               annotate_state_t **statep)
{
    int r = 0;

    if (statep) *statep = NULL;

    if (!mailbox->annot_state)
        mailbox->annot_state = annotate_state_new();

    r = annotate_state_set_message(mailbox->annot_state, mailbox, uid);
    if (r) return r;

    /* lock immediately if we have a write lock */
    if (mailbox_index_islocked(mailbox, /*write*/1))
        annotate_state_begin(mailbox->annot_state);

    if (statep) *statep = mailbox->annot_state;

    return 0;
}


static void mailbox_release_resources(struct mailbox *mailbox)
{
    int i;

    if (mailbox->i.dirty)
        abort();

    /* just close the header */
    xclose(mailbox->header_fd);

    /* release and unmap index */
    xclose(mailbox->index_fd);
    mailbox->index_locktype = 0; /* lock was released by closing fd */
    if (mailbox->index_base)
        map_free(&mailbox->index_base, &mailbox->index_len);

    /* release caches */
    for (i = 0; i < mailbox->caches.count; i++) {
        struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i);
        mappedfile_close(&cachefile);
    }
    ptrarray_fini(&mailbox->caches);
}


static int _store_change(struct mailbox *mailbox, struct index_record *record, int flags)
{
    struct index_change *change = _find_change(mailbox, record->recno);

    if (!change) {
        mailbox->index_change_count++;

        /* allocate a space if required */
        if (mailbox->index_change_count > mailbox->index_change_alloc) {
            mailbox->index_change_alloc += 256;
            mailbox->index_changes = xrealloc(mailbox->index_changes, sizeof(struct index_change) * mailbox->index_change_alloc);
        }

        change = &mailbox->index_changes[mailbox->index_change_count-1];
        memset(change, 0, sizeof(struct index_change));

        /* stitch into place */
        uint32_t pos = record->recno % INDEX_MAP_SIZE;
        change->mapnext = mailbox->index_change_map[pos];
        mailbox->index_change_map[pos] = mailbox->index_change_count; /* always non-zero */
    }

    /* finally always copy the data into place */
    change->record = *record;
    change->flags = flags;

    if (mailbox_cacherecord(mailbox, record)) {
        /* failed to load cache record */
        free(change->msgid);
        change->msgid = xstrdup("unknown");
    }
    else {
        char *c_env = xstrndup(cacheitem_base(record, CACHE_ENVELOPE) + 1,
                               cacheitem_size(record, CACHE_ENVELOPE) - 2);
        char *envtokens[NUMENVTOKENS];
        parse_cached_envelope(c_env, envtokens, NUMENVTOKENS);
        free(change->msgid);
        change->msgid = xstrdup(envtokens[ENV_MSGID] ? envtokens[ENV_MSGID] : "unknown");
        free(c_env);
    }

    annotate_state_t *astate = NULL;
    int r = mailbox_get_annotate_state(mailbox, record->uid, &astate);
    if (r) return r;

    struct buf annotval = BUF_INITIALIZER;
    if (record->cid && record->basecid && record->basecid != record->cid)
        buf_printf(&annotval, "%016llx", record->basecid);

    r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval);
    buf_free(&annotval);

    if (r) return r;

    return 0;
}


EXPORTED void mailbox_archive(struct mailbox *mailbox,
                              mailbox_decideproc_t *decideproc, void *deciderock, unsigned flags)

{
    int r;
    int dirtycache = 0;
    const message_t *msg;
    struct index_record copyrecord;
    const char *srcname;
    const char *destname;
    char *spoolcache = xstrdup(mailbox_meta_fname(mailbox, META_CACHE));
    char *archivecache = xstrdup(mailbox_meta_fname(mailbox, META_ARCHIVECACHE));
    int differentcache = strcmp(spoolcache, archivecache);
    int object_storage_enabled = 0;
#if defined ENABLE_OBJECTSTORE
    object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
#endif
    free(spoolcache);
    free(archivecache);

    assert(mailbox_index_islocked(mailbox, 1));
    if (!decideproc) decideproc = &mailbox_should_archive;

    struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, flags);

    while ((msg = mailbox_iter_step(iter))) {
        const struct index_record *record = msg_record(msg);
        const char *action = NULL;
        if (decideproc(mailbox, record, deciderock)) {
            if (record->internal_flags & FLAG_INTERNAL_ARCHIVED)
                continue;
            copyrecord = *record;
            srcname = mailbox_spool_fname(mailbox, copyrecord.uid);
            destname = mailbox_archive_fname(mailbox, copyrecord.uid);

            /* load cache before changing the flags */
            r = mailbox_cacherecord(mailbox, &copyrecord);
            if (r) {
                xsyslog(LOG_ERR, "IOERROR: failed to read cache",
                                 "mailbox=<%s> record=<%u> error=<%s>",
                                 mailbox_name(mailbox), copyrecord.uid, error_message(r));
                continue;
            }
#if defined ENABLE_OBJECTSTORE
            if (object_storage_enabled){
                /* upload on the blob store */
                r = objectstore_put(mailbox, &copyrecord, srcname);
                if (r) {
                    xsyslog(LOG_ERR, "IOERROR: objectstorage put failed",
                                     "mailbox=<%s> record=<%u> "
                                        "srcname=<%s> error=<%s>",
                                     mailbox_name(mailbox), copyrecord.uid,
                                     srcname, error_message(r));
                    // didn't manage to store it, so remove the ARCHIVED flag
                    continue;
                }
                r = unlink (srcname);
                if (r < 0)
                    syslog(LOG_ERR, "unlink(%s) failed: %m", srcname);
            }
#endif
            copyrecord.internal_flags |= FLAG_INTERNAL_ARCHIVED | FLAG_INTERNAL_NEEDS_CLEANUP;
            action = "archive";
        }
        else
        {
            if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED))
                continue;
            copyrecord = *record;
            destname = mailbox_spool_fname(mailbox, copyrecord.uid);
            srcname = mailbox_archive_fname(mailbox, copyrecord.uid);

            /* load cache before changing the flags */
            r = mailbox_cacherecord(mailbox, &copyrecord);
            if (r) {
                xsyslog(LOG_ERR, "IOERROR: failed to read cache",
                                 "mailbox=<%s> record=<%u> error=<%s>",
                                 mailbox_name(mailbox), copyrecord.uid, error_message(r));
                continue;
            }

#if defined ENABLE_OBJECTSTORE
            if (object_storage_enabled){
                /* recover from the blob store */
                r = objectstore_get(mailbox, &copyrecord, destname);
                if (r) {
                    xsyslog(LOG_ERR, "IOERROR: objectstorage get failed",
                                     "mailbox=<%s> record=<%u> "
                                        "destname=<%s> error=<%s>",
                                     mailbox_name(mailbox), copyrecord.uid,
                                     destname, error_message(r));
                    continue;
                }
                objectstore_delete(mailbox, &copyrecord); // this should only lower ref count.
            }
#endif

            copyrecord.internal_flags &= ~FLAG_INTERNAL_ARCHIVED;
            copyrecord.internal_flags |= FLAG_INTERNAL_NEEDS_CLEANUP;
            action = "unarchive";
        }

        if (!object_storage_enabled){
            /* got a file to copy! */
            if (strcmp(srcname, destname)) {
                r = cyrus_copyfile(srcname, destname, COPYFILE_MKDIR|COPYFILE_KEEPTIME);
                if (r) {
                    xsyslog(LOG_ERR, "IOERROR: copyfile failed",
                                     "mailbox=<%s> record=<%u> "
                                        "srcname=<%s> destname=<%s> error=<%s>",
                                     mailbox_name(mailbox), copyrecord.uid,
                                     srcname, destname, error_message(r));
                    continue;
                }
            }
        }

        /* got a new cache record to write */
        if (differentcache)
        {
            dirtycache = 1;
            copyrecord.cache_offset = 0;
            if (mailbox_append_cache(mailbox, &copyrecord))
                continue;
        }

        /* rewrite the index record */
        copyrecord.silentupdate = 1;
        copyrecord.ignorelimits = 1;
        if (mailbox_rewrite_index_record(mailbox, &copyrecord))
            continue;
        mailbox->i.options |= OPT_MAILBOX_NEEDS_UNLINK;

        if (config_auditlog) {
            char flagstr[FLAGMAPSTR_MAXLEN];
            flags_to_str(&copyrecord, flagstr);
            syslog(LOG_NOTICE, "auditlog: %s sessionid=<%s> mailbox=<%s> "
                   "uniqueid=<%s> uid=<%u> guid=<%s> cid=<%s> sysflags=<%s>",
                   action, session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox),
                   copyrecord.uid, message_guid_encode(&copyrecord.guid),
                   conversation_id_encode(copyrecord.cid), flagstr);
        }
    }
    mailbox_iter_done(&iter);

    /* if we have stale cache records, we'll need a repack */
    if (dirtycache) {
        mailbox_index_dirty(mailbox);
        mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK;
    }
}


static int mailbox_lock_index_internal(struct mailbox *mailbox, int locktype)
{
    struct stat sbuf;
    int r = 0;
    const char *header_fname = mailbox_meta_fname(mailbox, META_HEADER);
    const char *index_fname = mailbox_meta_fname(mailbox, META_INDEX);

    assert(mailbox->index_fd != -1);
    assert(!mailbox->index_locktype);

    char *userid = mboxname_to_userid(mailbox_name(mailbox));
    if (userid) {
        if (!user_isnamespacelocked(userid)) {
            struct mailboxlist *listitem = find_listitem(mailbox_name(mailbox));
            assert(listitem);
            assert(&listitem->m == mailbox);
            r = mailbox_mboxlock_reopen(listitem, LOCK_SHARED, locktype);
            if (locktype == LOCK_SHARED)
                mailbox->is_readonly = 1;
            if (!r) r = mailbox_open_index(mailbox);
        }
        free(userid);
        if (r) return r;
    }

    if (locktype == LOCK_EXCLUSIVE) {
        /* handle read-only case cleanly - we need to re-open read-write first! */
        if (mailbox->is_readonly) {
            mailbox->is_readonly = 0;
            r = mailbox_open_index(mailbox);
        }
        if (!r) r = lock_blocking(mailbox->index_fd, index_fname);
    }
    else if (locktype == LOCK_SHARED) {
        r = lock_shared(mailbox->index_fd, index_fname);
    }
    else {
        /* this function does not support nonblocking locks */
        fatal("invalid locktype for index", EX_SOFTWARE);
    }

    /* double check that the index exists and has at least enough
     * data to check the version number */
    if (!r) {
        if (!mailbox->index_base)
            r = IMAP_MAILBOX_BADFORMAT;
        else if (mailbox->index_size < OFFSET_NUM_RECORDS)
            r = IMAP_MAILBOX_BADFORMAT;
        if (r)
            lock_unlock(mailbox->index_fd, index_fname);
    }

    if (r) {
        xsyslog(LOG_ERR, "IOERROR: lock index failed",
                         "mailbox=<%s> error=<%s>",
                         mailbox_name(mailbox), error_message(r));
        return IMAP_IOERROR;
    }

    mailbox->index_locktype = locktype;
    gettimeofday(&mailbox->starttime, 0);

    r = stat(header_fname, &sbuf);
    if (r == -1) {
        xsyslog(LOG_ERR, "IOERROR: stat header failed",
                         "mailbox=<%s> header=<%s>",
                         mailbox_name(mailbox), header_fname);
        mailbox_unlock_index(mailbox, NULL);
        return IMAP_IOERROR;
    }

    /* has the header file changed? */
    if (sbuf.st_ino != mailbox->header_file_ino) {
        r = mailbox_read_header(mailbox, NULL);
        if (r) {
            xsyslog(LOG_ERR, "IOERROR: read header failed",
                             "mailbox=<%s> error=<%s>",
                             mailbox_name(mailbox), error_message(r));
            mailbox_unlock_index(mailbox, NULL);
            return r;
        }
    }

    /* release caches */
    int i;
    for (i = 0; i < mailbox->caches.count; i++) {
        struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i);
        mappedfile_close(&cachefile);
    }
    ptrarray_fini(&mailbox->caches);

    /* note: it's guaranteed by our outer cyrus.lock lock that the
     * cyrus.index and cyrus.cache files are never rewritten, so
     * we're safe to just extend the map if needed */
    r = mailbox_read_index_header(mailbox);
    if (r) {
        xsyslog(LOG_ERR, "IOERROR: refreshing index failed",
                         "mailbox=<%s> error=<%s>",
                         mailbox_name(mailbox), error_message(r));
        mailbox_unlock_index(mailbox, NULL);
        return r;
    }

    /* check the CRC */
    if (mailbox->header_file_crc && mailbox->i.header_file_crc &&
        mailbox->header_file_crc != mailbox->i.header_file_crc) {
        syslog(LOG_WARNING, "Header CRC mismatch for mailbox %s: %08X %08X",
               mailbox_name(mailbox), (unsigned int)mailbox->header_file_crc,
               (unsigned int)mailbox->i.header_file_crc);
    }

    return 0;
}


static int mailbox_delete_alarms(struct mailbox *mailbox)
{
    if (!(mailbox->i.options & OPT_IMAP_HAS_ALARMS))
        return 0;

    return caldav_alarm_delete_mailbox(mailbox_name(mailbox));
}


static int cache_append_record(struct mappedfile *mf, struct index_record *record)
{
    const struct buf *buf = cache_buf(record);
    size_t offset = mappedfile_size(mf);
    int n;

    n = mappedfile_pwritebuf(mf, buf, offset);
    if (n < 0) {
        syslog(LOG_ERR, "failed to append " SIZE_T_FMT " bytes to cache", buf->len);
        return IMAP_IOERROR;
    }

    record->cache_offset = offset;

    return 0;
}


static int do_reconstruct(struct findall_data *data, void *rock)
{
    if (!data) return 0;
    struct reconstruct_rock *rrock = (struct reconstruct_rock *) rock;
    int r;
    struct mailbox *mailbox = NULL;
    char outpath[MAX_MAILBOX_PATH];
    const char *name = NULL;
    int make_changes = reconstruct_flags & RECONSTRUCT_MAKE_CHANGES;
    int prefer_mbentry = reconstruct_flags & RECONSTRUCT_PREFER_MBOXLIST;

    /* ignore partial matches */
    if (!data->is_exactmatch) return 0;

    /* ignore intermediates */
    if ((data->mbentry->mbtype & MBTYPE_INTERMEDIATE))
        return 0;
    /* ignore remote */
    if ((data->mbentry->mbtype & MBTYPE_REMOTE))
        return 0;

    signals_poll();

    name = mbname_intname(data->mbname);

    /* don't repeat */
    if (hash_lookup(name, &rrock->visited)) return 0;

    struct mboxlock *namespacelock = mboxname_usernamespacelock(name);

    if (setversion) {
        r = mailbox_open_iwl(name, &mailbox);
        if (r) {
            com_err(name, r, "Failed to open mailbox to set version");
            mboxname_release(&namespacelock);
            return 0;
        }
    }
    else {
        r = mailbox_reconstruct(name, reconstruct_flags, &mailbox);
        if (r) {
            com_err(name, r, "%s",
                    (r == IMAP_IOERROR) ? error_message(errno) : "Failed to reconstruct mailbox");
            mboxname_release(&namespacelock);
            return 0;
        }
    }

    mbentry_t *mbentry_byid = NULL;
    mbentry_t *mbentry_byname = NULL;

    r = mboxlist_lookup(name, &mbentry_byname, NULL);
    // this better succeed!
    if (r) {
        printf("Failed to re-read %s!\n", name);
        exit(1);
    }

    int mbentry_dirty = 0;

    // fix any uniqueid related mixups first!
    if (strcmpsafe(mailbox_uniqueid(mailbox), mbentry_byname->uniqueid)) {
        printf("Wrong uniqueid in mbentry, fixing %s (%s -> %s)\n",
               name, mbentry_byname->uniqueid, mailbox_uniqueid(mailbox));
        xzfree(mbentry_byname->uniqueid);
        mbentry_byname->uniqueid = xstrdupnull(mailbox_uniqueid(mailbox));
        mbentry_dirty = 1;
    }

    // fetch by uniqueid and compare
    r = mboxlist_lookup_by_uniqueid(mbentry_byname->uniqueid, &mbentry_byid, NULL);
    if (r == IMAP_MAILBOX_NONEXISTENT) {
        printf("Missing mboxlist entry for uniqueid - will rewrite %s (%s)\n",
               name, mbentry_byname->uniqueid);
        mbentry_dirty = 1;
    }
    else if (r) {
        printf("Error reading mailboxesdb for %s\n", name);
        exit(1);
    }
    else if (strcmpsafe(mbentry_byid->name, name)) {
        printf("Wrong uniqueid! %s (should be %s)\n", mbentry_byid->name, name);
        if (updateuniqueids) {
            mailbox_make_uniqueid(mailbox);
            xzfree(mbentry_byname->uniqueid);
            mbentry_byname->uniqueid = xstrdupnull(mailbox_uniqueid(mailbox));
            mbentry_dirty = 1;
            syslog (LOG_ERR, "uniqueid clash with %s - changed %s (%s => %s)",
                    mbentry_byid->name, mailbox_name(mailbox), mbentry_byid->uniqueid, mailbox_uniqueid(mailbox));
        }
        else {
            syslog (LOG_ERR, "uniqueid clash with %s for %s (%s)",
                    mbentry_byid->name, mailbox_name(mailbox), mailbox_uniqueid(mailbox));
            exit(1);
        }
    }
    else {
        // see if anything is actually different
        if (mbentry_byname->uidvalidity != mbentry_byid->uidvalidity) {
            printf("mismatched uidvalidity byid %s (%u %u)\n",
                   name, mbentry_byname->uidvalidity, mbentry_byid->uidvalidity);
            mbentry_dirty = 1;
        }
        if (mbentry_byname->mbtype != mbentry_byid->mbtype) {
            printf("mismatched mbtype byid %s (%u %u)\n",
                   name, mbentry_byname->mbtype, mbentry_byid->mbtype);
            mbentry_dirty = 1;
        }
        if (strcmpsafe(mbentry_byname->partition, mbentry_byid->partition)) {
            printf("mismatched partition byid %s (%s %s)\n",
                   name, mbentry_byname->partition, mbentry_byid->partition);
            mbentry_dirty = 1;
        }
        if (strcmpsafe(mbentry_byname->server, mbentry_byid->server)) {
            printf("mismatched server byid %s (%s %s)\n",
                   name, mbentry_byname->server, mbentry_byid->server);
            mbentry_dirty = 1;
        }
        if (strcmpsafe(mbentry_byname->acl, mbentry_byid->acl)) {
            printf("mismatched acl byid %s (%s %s)\n",
                   name, mbentry_byname->acl, mbentry_byid->acl);
            mbentry_dirty = 1;
        }
    }
    mboxlist_entry_free(&mbentry_byid);

    if (mailbox->i.uidvalidity != mbentry_byname->uidvalidity) {
        if (!mailbox->i.uidvalidity || (prefer_mbentry && mbentry_byname->uidvalidity)) {
            printf("Wrong uidvalidity in mailbox, fixing %s (%u -> %u)\n",
                   name, mailbox->i.uidvalidity, mbentry_byname->uidvalidity);
            mailbox->i.uidvalidity = mbentry_byname->uidvalidity;
            mailbox_index_dirty(mailbox);
        }
        else {
            printf("Wrong uidvalidity in mbentry, fixing %s (%u -> %u)\n",
                   name, mbentry_byname->uidvalidity, mailbox->i.uidvalidity);
            mbentry_byname->uidvalidity = mailbox->i.uidvalidity;
            mbentry_dirty = 1;
        }
    }

    if (strcmpsafe(mailbox_acl(mailbox), mbentry_byname->acl)) {
        if (prefer_mbentry) {
            printf("Wrong acl in mbentry %s (%s %s)\n",
                   name, mailbox_acl(mailbox), mbentry_byname->acl);
            // this sets the header to dirty
            mailbox_set_acl(mailbox, mbentry_byname->acl);
        }
        else {
            printf("Wrong acl in mbentry %s (%s %s)\n",
                   name, mbentry_byname->acl, mailbox_acl(mailbox));
            xzfree(mbentry_byname->acl);
            mbentry_byname->acl = xstrdupnull(mailbox_acl(mailbox));
            mbentry_dirty = 1;
        }
    }

    if (mbentry_dirty && make_changes) {
        r = mboxlist_update(mbentry_byname, 1);
        if (r) syslog(LOG_ERR, "IOERROR: failed to update mbentry for %s (%s)", name, error_message(r));
    }
    mboxlist_entry_free(&mbentry_byname);

    /* Convert internal name to external */
    char *extname = mboxname_to_external(name, &recon_namespace, NULL);
    if (!(reconstruct_flags & RECONSTRUCT_QUIET))
        printf("%s\n", extname);

    strncpy(outpath, mailbox_meta_fname(mailbox, META_HEADER), MAX_MAILBOX_NAME);

    if (setversion && setversion != mailbox->i.minor_version) {
        int oldversion = mailbox->i.minor_version;
        /* need to re-set the version! */
        int r = mailbox_setversion(mailbox, setversion);
        if (r) {
            printf("FAILED TO REPACK %s with new version %s\n", extname, error_message(r));
        }
        else {
            printf("Converted %s version %d to %d\n", extname, oldversion, setversion);
        }
    }
    if (make_changes) {
        mailbox_commit(mailbox);
    }
    else {
        mailbox_abort(mailbox);
    }
    mailbox_close(&mailbox);
    free(extname);
    mboxname_release(&namespacelock);

    if (rrock->discovered) {
        char fnamebuf[MAX_MAILBOX_PATH];
        char *ptr;
        DIR *dirp;
        struct dirent *dirent;
        struct stat sbuf;

        ptr = strstr(outpath, "cyrus.header");
        if (!ptr) return 0;
        *ptr = 0;

        r = chdir(outpath);
        if (r) return 0;

        /* we recurse down this directory to see if there's any mailboxes
           under this not in the mailboxes database */
        dirp = opendir(".");
        if (!dirp) return 0;

        while ((dirent = readdir(dirp)) != NULL) {
            /* mailbox directories never have a dot in them */
            if (strchr(dirent->d_name, '.')) continue;
            if (stat(dirent->d_name, &sbuf) < 0) continue;
            if (!S_ISDIR(sbuf.st_mode)) continue;

            /* ok, we found a directory that doesn't have a dot in it;
               is there a cyrus.header file? */
            snprintf(fnamebuf, MAX_MAILBOX_PATH, "%s%s",
                     dirent->d_name, FNAME_HEADER);
            if (stat(fnamebuf, &sbuf) < 0) continue;

            /* ok, we have a real mailbox directory */
            char buf[MAX_MAILBOX_NAME];
            snprintf(buf, MAX_MAILBOX_NAME, "%s.%s",
                     name, dirent->d_name);

            /* does fnamebuf exist as a mailbox in mboxlist? */
            do {
                r = mboxlist_lookup(buf, NULL, NULL);
            } while (r == IMAP_AGAIN);
            if (!r) continue; /* mailbox exists; it'll be reconstructed
                                 with a -r */

            if (r != IMAP_MAILBOX_NONEXISTENT) break; /* erg? */
            else r = 0; /* reset error condition */

            printf("discovered %s\n", buf);
            strarray_append(rrock->discovered, buf);
        }
        closedir(dirp);
    }

    /* mark it as visited
     * we don't care about the value, it just needs to be a non-NULL pointer
     */
    hash_insert(name, &rrock, &rrock->visited);

    return 0;
}


static int mailbox_refresh_index_map(struct mailbox *mailbox)
{
    size_t need_size;
    struct stat sbuf;

    /* check if we need to extend the mmaped space for the index file
     * (i.e. new records appended since last read) */
    need_size = mailbox->i.start_offset +
                mailbox->i.num_records * mailbox->i.record_size;
    if (mailbox->index_size < need_size) {
        if (fstat(mailbox->index_fd, &sbuf) == -1)
            return IMAP_IOERROR;

        if (sbuf.st_size < (int)need_size)
            return IMAP_MAILBOX_BADFORMAT;

        mailbox->index_size = sbuf.st_size;

    }

    /* always refresh, we may be using map_nommap */
    map_refresh(mailbox->index_fd, 1, &mailbox->index_base,
                &mailbox->index_len, mailbox->index_size,
                "index", mailbox_name(mailbox));

    return 0;
}


EXPORTED int mailbox_index_islocked(struct mailbox *mailbox, int write)
{
    if (mailbox->index_locktype == LOCK_EXCLUSIVE) return 1;
    if (mailbox->index_locktype == LOCK_SHARED && !write) return 1;
    return 0;
}


static int mailbox_is_virtannot(struct mailbox *mailbox, const char *entry)
{
    if (mailbox->i.minor_version < 13) return 0;
    // thrid was introduced in v13
    if (!strcmp(entry, IMAP_ANNOT_NS "thrid")) return 1;

    if (mailbox->i.minor_version < 15) return 0;
    // savedate was introduced in v15
    if (!strcmp(entry, IMAP_ANNOT_NS "savedate")) return 1;

    if (mailbox->i.minor_version < 16) return 0;
    // createdmodseq was introduced in v16
    if (!strcmp(entry, IMAP_ANNOT_NS "createdmodseq")) return 1;

    return 0;
}


static void mailbox_record_cleanup(struct mailbox *mailbox,
                                   struct index_record *record)
{
#if defined ENABLE_OBJECTSTORE
    if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)) {
        /* we always remove the spool file here, because we've archived it */
        if (record->system_flags & FLAG_INTERNAL_ARCHIVED)
            unlink(spoolfname);

        /* if the record is also deleted, we remove the objectstore copy */
        if (record->system_flags & FLAG_INTERNAL_UNLINKED)
            objectstore_delete(mailbox, record);

        return;
    }
#endif

    if (record->internal_flags & FLAG_INTERNAL_UNLINKED) {
        char flagstr[FLAGMAPSTR_MAXLEN];
        flags_to_str(record, flagstr);

        /* remove both files */
        mailbox_cleanup_uid(mailbox, record->uid, flagstr);

        int r = mailbox_get_annotate_state(mailbox, record->uid, NULL);
        if (r) {
            xsyslog(LOG_ERR, "IOERROR: failed to open annotations",
                             "mailbox=<%s> record=<%u> error=<%s>",
                             mailbox_name(mailbox), record->uid, error_message(r));
        }

        r = annotate_msg_cleanup(mailbox, record->uid);
        if (r) {
            xsyslog(LOG_ERR, "IOERROR: failed to cleanup annotations",
                             "mailbox=<%s> record=<%u> error=<%s>",
                             mailbox_name(mailbox), record->uid, error_message(r));
        }

        return;
    }

    /* file is still alive - check if there's anything to clean up */

    const char *spoolfname = mailbox_spool_fname(mailbox, record->uid);
    const char *archivefname = mailbox_archive_fname(mailbox, record->uid);

    /* don't cleanup if it's the same file! */
    if (strcmp(spoolfname, archivefname)) {
        if (record->internal_flags & FLAG_INTERNAL_ARCHIVED) {
            /* XXX - stat to make sure the other file exists first? - we mostly
            *  trust that we didn't do stupid things everywhere else, so maybe not */
            unlink(spoolfname);
        }

        else {
            unlink(archivefname);
        }
    }
}


EXPORTED int mailbox_create(const char *name,
                   uint32_t mbtype,
                   const char *part,
                   const char *acl,
                   const char *uniqueid,
                   int options,
                   unsigned uidvalidity,
                   modseq_t createdmodseq,
                   modseq_t highestmodseq,
                   struct mailbox **mailboxptr)
{
    int r = 0;
    char quotaroot[MAX_MAILBOX_BUFFER];
    int hasquota;
    const char *fname;
    struct mailbox *mailbox = NULL;
    int n;
    int createfnames[] = { META_INDEX, META_HEADER, 0 };
    struct mailboxlist *listitem;

    assert(uniqueid);

    /* if we already have this name open then that's an error too */
    listitem = find_listitem(name);
    if (listitem) return IMAP_MAILBOX_LOCKED;

    listitem = create_listitem(name);
    mailbox = &listitem->m;

    /* needs to be an exclusive namelock to create a mailbox */
    char *userid = mboxname_to_userid(name);
    if (userid) {
        int haslock = user_isnamespacelocked(userid);
        assert(haslock == LOCK_EXCLUSIVE);
        free(userid);
    }

    uint32_t legacy_dirs = (mbtype & MBTYPE_LEGACY_DIRS);
    r = mboxname_lock(legacy_dirs ? name : uniqueid, &listitem->l, LOCK_EXCLUSIVE);
    if (r) {
        if (mailbox->local_namespacelock)
            mboxname_release(&mailbox->local_namespacelock);
        remove_listitem(listitem);
        return r;
    }

    // fill out the initial mbentry (XXX: pass it in from mboxlist.c)
    mailbox->mbentry = mboxlist_entry_create();
    mailbox->mbentry->name = xstrdup(name);
    mailbox->mbentry->partition = xstrdup(part);
    mailbox->mbentry->acl = xstrdup(acl);
    mailbox->mbentry->mbtype = mbtype;
    mailbox->mbentry->uniqueid = xstrdup(uniqueid);

    // fill out the header too
    mailbox->h.name = xstrdup(name);
    mailbox->h.acl = xstrdup(acl);
    mailbox->h.mbtype = mbtype;
    mailbox->h.uniqueid = xstrdup(uniqueid);

    // if we've been given a highestmodseq, we don't update it
    if (highestmodseq) mailbox->silentchanges = 1;

    // set the quotaroot if any
    hasquota = quota_findroot(quotaroot, sizeof(quotaroot), name);
    if (hasquota) {
        mailbox->h.quotaroot = xstrdup(quotaroot);
        mailbox->quota_dirty = 1;
    }

    /* ensure all paths exist */
    for (n = 0; createfnames[n]; n++) {
        fname = mailbox_meta_fname(mailbox, createfnames[n]);
        if (!fname) {
            xsyslog(LOG_ERR, "IOERROR: Mailbox name too long",
                             "mailbox=<%s>",
                             mailbox_name(mailbox));
            r = IMAP_MAILBOX_BADNAME;
            goto done;
        }
        if (cyrus_mkdir(fname, 0755) == -1) {
            xsyslog(LOG_ERR, "IOERROR: mkdir failed",
                             "fname=<%s>",
                             fname);
            r = IMAP_IOERROR;
            goto done;
        }
    }

    /* ensure we can fit the longest possible file name */
    fname = mailbox_datapath(mailbox, 0);
    if (!fname) {
        xsyslog(LOG_ERR, "IOERROR: Mailbox name too long",
                         "mailbox=<%s>",
                         mailbox_name(mailbox));
        r = IMAP_MAILBOX_BADNAME;
        goto done;
    }
    /* and create the directory too :) */
    if (cyrus_mkdir(fname, 0755) == -1) {
        xsyslog(LOG_ERR, "IOERROR: mkdir failed",
                         "fname=<%s>",
                         fname);
        r = IMAP_IOERROR;
        goto done;
    }

    fname = mailbox_meta_fname(mailbox, META_INDEX);
    if (!fname) {
        xsyslog(LOG_ERR, "IOERROR: Mailbox name too long",
                         "mailbox=<%s>",
                         mailbox_name(mailbox));
        r = IMAP_MAILBOX_BADNAME;
        goto done;
    }
    mailbox->index_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
    if (mailbox->index_fd == -1) {
        xsyslog(LOG_ERR, "IOERROR: create index failed",
                         "fname=<%s>",
                         fname);
        r = IMAP_IOERROR;
        goto done;
    }
    r = lock_blocking(mailbox->index_fd, fname);
    if (r) {
        xsyslog(LOG_ERR, "IOERROR: lock index failed",
                         "fname=<%s>",
                         fname);
        r = IMAP_IOERROR;
        goto done;
    }
    mailbox->index_locktype = LOCK_EXCLUSIVE;

    /* ensure a UIDVALIDITY is set */
    if (!uidvalidity)
        uidvalidity = mboxname_nextuidvalidity(name, time(0));
    else
        mboxname_setuidvalidity(mailbox_name(mailbox), uidvalidity);
    mailbox->mbentry->uidvalidity = uidvalidity;

    /* and highest modseq */
    if (!highestmodseq)
        highestmodseq = mboxname_nextmodseq(mailbox_name(mailbox), 0, mbtype,
                                            MBOXMODSEQ_ISFOLDER);
    else
        mboxname_setmodseq(mailbox_name(mailbox), highestmodseq, mbtype,
                           MBOXMODSEQ_ISFOLDER);

    /* and created modseq */
    if (!createdmodseq || createdmodseq > highestmodseq)
        createdmodseq = highestmodseq;

    /* init non-zero fields */
    mailbox_index_dirty(mailbox);
    mailbox->i.minor_version = MAILBOX_MINOR_VERSION;
    mailbox->i.start_offset = INDEX_HEADER_SIZE;
    mailbox->i.record_size = INDEX_RECORD_SIZE;
    mailbox->i.options = options;
    mailbox->i.uidvalidity = uidvalidity;
    mailbox->i.createdmodseq = createdmodseq;
    mailbox->i.highestmodseq = highestmodseq;
    mailbox->i.changes_epoch = time(0);
    mailbox->i.synccrcs.basic = CRC_INIT_BASIC;
    mailbox->i.synccrcs.annot = CRC_INIT_ANNOT;

    /* initialise header size field so appends calculate the
     * correct map size */
    mailbox->index_size = INDEX_HEADER_SIZE;

    mailbox->header_dirty = 1;

    r = seen_create_mailbox(NULL, mailbox);
    if (r) goto done;
    r = mailbox_commit(mailbox);
    if (r) goto done;

    if (config_auditlog)
        syslog(LOG_NOTICE, "auditlog: create sessionid=<%s> "
                           "mailbox=<%s> uniqueid=<%s> uidvalidity=<%u>",
                           session_id(), mailbox_name(mailbox),
                           mailbox_uniqueid(mailbox), mailbox->i.uidvalidity);

done:
    if (!r && mailboxptr)
        *mailboxptr = mailbox;
    else
        mailbox_close(&mailbox);

    return r;
}


EXPORTED void mailbox_remove_files_from_object_storage(struct mailbox *mailbox,
                              unsigned flags)

{
    const message_t *msg;
#if defined ENABLE_OBJECTSTORE
    int object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
#endif

    assert(mailbox_index_islocked(mailbox, 1));
    struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, flags);
    while ((msg = mailbox_iter_step(iter))) {
        const struct index_record *record = msg_record(msg);
        if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED))
            continue;
#if defined ENABLE_OBJECTSTORE
        if (object_storage_enabled)
            objectstore_delete(mailbox, record);  // this should only lower ref count.
#endif
    }
    mailbox_iter_done(&iter);
}


static int load_annot_cb(const char *mailbox __attribute__((unused)),
                         uint32_t uid __attribute__((unused)),
                         const char *entry, const char *userid,
                         const struct buf *value,
                         const struct annotate_metadata *mdata __attribute__((unused)),
                         void *rock)
{
    struct entryattlist **eal = (struct entryattlist **)rock;
    const char *attrib = (userid[0] ? "value.priv" : "value.shared");
    setentryatt(eal, entry, attrib, value);
    return 0;
}


EXPORTED void mailbox_set_wait_cb(void (*cb)(void *), void *rock)
{
    mailbox_wait_cb = cb;
    mailbox_wait_cb_rock = rock;
}


static void get_partition(const char *key, const char *val, void *rock)
{
    struct part_rock *prock = (struct part_rock *) rock;

    if (*prock->partition) return;

    if (prock->is_meta) {
        if (strncmp("meta", key, 4)) return;
        key += 4;
    }
    if (strncmp("partition-", key, 10)) return;

    size_t vlen = strlen(val);
    if (!strncmp(prock->path, val, vlen) &&
        (val[vlen-1] == '/' || prock->path[vlen] == '/')) {
        *prock->partition = xstrdup(key+10);
    }
}


static int mailbox_buf_to_index_header(const char *buf, size_t len,
                                       struct index_header *i)
{
    uint32_t crc;
    bit32 qannot;
    size_t headerlen;

    if (len < OFFSET_MINOR_VERSION+4)
        return IMAP_MAILBOX_BADFORMAT;

    memset(i, 0, sizeof(struct index_header));

    i->generation_no = ntohl(*((bit32 *)(buf+OFFSET_GENERATION_NO)));
    i->format = ntohl(*((bit32 *)(buf+OFFSET_FORMAT)));
    i->minor_version = ntohl(*((bit32 *)(buf+OFFSET_MINOR_VERSION)));
    switch (i->minor_version) {
    case 6:
    case 7:
        headerlen = 76;
        break;
    case 8:
        headerlen = 92;
        break;
    case 9:
    case 10:
        headerlen = 96;
        break;
    case 12:
    case 13:
        headerlen = 128;
        break;
    case 14:
    case 15:
    case 16:
    case 17:
        headerlen = 160;
        break;
    default:
        return IMAP_MAILBOX_BADFORMAT;
    }
    if (len < headerlen)
        return IMAP_MAILBOX_BADFORMAT;

    i->start_offset = ntohl(*((bit32 *)(buf+OFFSET_START_OFFSET)));
    i->record_size = ntohl(*((bit32 *)(buf+OFFSET_RECORD_SIZE)));
    i->num_records = ntohl(*((bit32 *)(buf+OFFSET_NUM_RECORDS)));
    i->last_appenddate = ntohl(*((bit32 *)(buf+OFFSET_LAST_APPENDDATE)));
    i->last_uid = ntohl(*((bit32 *)(buf+OFFSET_LAST_UID)));
    i->quota_mailbox_used = align_ntohll(buf+OFFSET_QUOTA_MAILBOX_USED);
    i->pop3_last_login = ntohl(*((bit32 *)(buf+OFFSET_POP3_LAST_LOGIN)));
    i->uidvalidity = ntohl(*((bit32 *)(buf+OFFSET_UIDVALIDITY)));
    i->deleted = ntohl(*((bit32 *)(buf+OFFSET_DELETED)));
    i->answered = ntohl(*((bit32 *)(buf+OFFSET_ANSWERED)));
    i->flagged = ntohl(*((bit32 *)(buf+OFFSET_FLAGGED)));
    i->options = ntohl(*((bit32 *)(buf+OFFSET_MAILBOX_OPTIONS)));
    i->leaked_cache_records = ntohl(*((bit32 *)(buf+OFFSET_LEAKED_CACHE)));
    if (i->minor_version < 8) goto done;

    i->highestmodseq = align_ntohll(buf+OFFSET_HIGHESTMODSEQ);
    if (i->minor_version < 12) goto done;

    i->deletedmodseq = align_ntohll(buf+OFFSET_DELETEDMODSEQ);
    i->exists = ntohl(*((bit32 *)(buf+OFFSET_EXISTS)));
    i->first_expunged = ntohl(*((bit32 *)(buf+OFFSET_FIRST_EXPUNGED)));
    i->last_repack_time = ntohl(*((bit32 *)(buf+OFFSET_LAST_REPACK_TIME)));
    i->header_file_crc = ntohl(*((bit32 *)(buf+OFFSET_HEADER_FILE_CRC)));
    i->synccrcs.basic = ntohl(*((bit32 *)(buf+OFFSET_SYNCCRCS_BASIC)));
    i->recentuid = ntohl(*((bit32 *)(buf+OFFSET_RECENTUID)));
    i->recenttime = ntohl(*((bit32 *)(buf+OFFSET_RECENTTIME)));

    if (i->minor_version < 13) goto crc;

    i->pop3_show_after = ntohl(*((bit32 *)(buf+OFFSET_POP3_SHOW_AFTER)));
    qannot = ntohl(*((bit32 *)(buf+OFFSET_QUOTA_ANNOT_USED)));
    /* this field is stored as a 32b unsigned on disk but 64b signed
     * in memory, so we need to be careful about sign extension */
    i->quota_annot_used = (quota_t)((unsigned long long)qannot);
    i->synccrcs.annot = ntohl(*((bit32 *)(buf+OFFSET_SYNCCRCS_ANNOT)));

    if (i->minor_version < 14) goto crc;

    i->unseen = ntohl(*((bit32 *)(buf+OFFSET_UNSEEN)));

    if (i->minor_version < 16) goto crc;

    i->createdmodseq = align_ntohll(buf+OFFSET_MAILBOX_CREATEDMODSEQ);

    if (i->minor_version < 17) goto crc;

    i->changes_epoch = ntohl(*((bit32 *)(buf+OFFSET_CHANGES_EPOCH)));

crc:
    /* CRC is always the last 4 bytes */
    crc = ntohl(*((bit32 *)(buf+headerlen-4)));
    if (crc != crc32_map(buf, headerlen-4))
        return IMAP_MAILBOX_CHECKSUM;

done:
    if (!i->exists)
        i->options |= OPT_POP3_NEW_UIDL;

    if (!i->highestmodseq)
        i->highestmodseq = 1;

    if (i->minor_version < 12) {
        i->deletedmodseq = i->highestmodseq;
        i->exists = i->num_records;
    }

    if (i->minor_version < 17) {
        i->changes_epoch = i->last_repack_time ? i->last_repack_time : time(0);
    }

    return 0;
}


static uint32_t crc_basic(const struct mailbox *mailbox,
                          const struct index_record *record)
{
    char buf[4096];
    uint32_t flagcrc = 0;
    int flag;

    /* expunged flags have no sync CRC */
    if (record->internal_flags & FLAG_INTERNAL_EXPUNGED)
        return 0;

    /* calculate an XORed CRC32 over all the flags on the message, so no
     * matter what order they are store in the header, the final value
     * is the same */
    if (record->system_flags & FLAG_DELETED)
        flagcrc ^= crc32_cstring("\\deleted");
    if (record->system_flags & FLAG_ANSWERED)
        flagcrc ^= crc32_cstring("\\answered");
    if (record->system_flags & FLAG_FLAGGED)
        flagcrc ^= crc32_cstring("\\flagged");
    if (record->system_flags & FLAG_DRAFT)
        flagcrc ^= crc32_cstring("\\draft");
    if (record->system_flags & FLAG_SEEN)
        flagcrc ^= crc32_cstring("\\seen");

    for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
        if (!mailbox->h.flagname[flag])
            continue;
        if (!(record->user_flags[flag/32] & (1<<(flag&31))))
            continue;
        /* need to compare without case being significant */
        strlcpy(buf, mailbox->h.flagname[flag], 4096);
        lcase(buf);
        flagcrc ^= crc32_cstring(buf);
    }

    snprintf(buf, sizeof(buf), "%u " MODSEQ_FMT " " TIME_T_FMT " (%u) " TIME_T_FMT " %s",
            record->uid, record->modseq, record->last_updated,
            flagcrc,
            record->internaldate,
            message_guid_encode(&record->guid));

    return crc32_cstring(buf);
}


static struct mappedfile *cache_getfile(ptrarray_t *list, const char *fname,
                                        int readonly, uint32_t generation)
{
    struct mappedfile *cachefile = NULL;
    int openflags = readonly ? 0 : MAPPEDFILE_CREATE | MAPPEDFILE_RW;
    int i;

    for (i = 0; i < list->count; i++) {
        cachefile = ptrarray_nth(list, i);
        if (!strcmp(fname, mappedfile_fname(cachefile)))
            return cachefile;
    }

    /* guess we didn't find it - open a new one */
    cachefile = NULL;
    if (mappedfile_open(&cachefile, fname, openflags)) {
        xsyslog(LOG_ERR, "IOERROR: failed to open cache file",
                         "fname=<%s>",
                         fname);
        return NULL;
    }

    if (!readonly && !mappedfile_size(cachefile)) {
        /* zero byte file?  Set the generation */
        uint32_t netgen = htonl(generation);
        mappedfile_pwrite(cachefile, (const char *) &netgen, 4, 0);
        mappedfile_commit(cachefile);
    }

    ptrarray_append(list, cachefile);

    return cachefile;
}


static int reconstruct_delannots(struct mailbox *mailbox,
                                 struct found_uids *delannots,
                                 int flags)
{
    int make_changes = (flags & RECONSTRUCT_MAKE_CHANGES);
    int r = 0;

    r = mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, NULL);
    if (r) {
        syslog(LOG_ERR, "IOERROR: failed to open annotations %s: %s",
               mailbox_name(mailbox), error_message(r));
        goto out;
    }

    while (delannots->pos < delannots->nused) {
        uint32_t uid = delannots->found[delannots->pos].uid;
        syslog(LOG_NOTICE, "removing stale annotations for %u", uid);
        printf("removing stale annotations for %u\n", uid);
        if (make_changes) {
            r = annotate_msg_cleanup(mailbox, uid);
            if (r) goto out;
        }
        delannots->pos++;
    }

out:
    return r;
}


static void remove_listitem(struct mailboxlist *remitem)
{
    struct mailboxlist *item;
    struct mailboxlist *previtem = NULL;

    for (item = open_mailboxes; item; item = item->next) {
        if (item == remitem) {

            if (previtem)
                previtem->next = item->next;
            else
                open_mailboxes = item->next;
            free(item->name);
            free(item);

#if defined ENABLE_OBJECTSTORE
            if (!open_mailboxes && config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED))  // time to close the database
                 keep_user_message_db_open (0);
#endif
            return;
        }
        previtem = item;
    }

    fatal("didn't find item in list", EX_SOFTWARE);
}


EXPORTED const char *mailbox_record_fname(struct mailbox *mailbox,
                                          const struct index_record *record)
{
    int object_storage_enabled = 0 ;
#if defined ENABLE_OBJECTSTORE
    object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ;
#endif

    if (!object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))
        return mailbox_archive_fname(mailbox, record->uid);
    else
        return mailbox_spool_fname(mailbox, record->uid);
}


HIDDEN int mailbox_commit_quota(struct mailbox *mailbox)
{
    int res;
    int changed = 0;
    quota_t quota_usage[QUOTA_NUMRESOURCES];

    /* not dirty */
    if (!mailbox->quota_dirty)
        return 0;

    mailbox->quota_dirty = 0;

    /* no quota root means we don't track quota.  That's OK */
    if (!mailbox_quotaroot(mailbox))
        return 0;

    mailbox_get_usage(mailbox, quota_usage);
    for (res = 0; res < QUOTA_NUMRESOURCES; res++) {
        quota_usage[res] -= mailbox->quota_previously_used[res];
        if (quota_usage[res] != 0) {
            changed++;
        }
    }
    /* unchanged */
    if (!changed)
        return 0;

    assert(mailbox_index_islocked(mailbox, 1));

    quota_update_useds(mailbox_quotaroot(mailbox), quota_usage,
                       mailbox_name(mailbox), mailbox->silentchanges);
    /* XXX - fail upon issue?  It's tempting */

    return 0;
}


static int mailbox_mboxlock_reopen(struct mailboxlist *listitem, int locktype, int index_locktype)
{
    struct mailbox *mailbox = &listitem->m;
    uint32_t legacy_dirs = (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS);
    int r;

    mailbox_release_resources(mailbox);

    mboxname_release(&listitem->l);
    mboxname_release(&mailbox->local_namespacelock);

    char *userid = mboxname_to_userid(mailbox_name(mailbox));
    if (userid) {
        int haslock = user_isnamespacelocked(userid);
        if (haslock) {
            if (index_locktype != LOCK_SHARED) assert(haslock != LOCK_SHARED);
        }
        else {
            mailbox->local_namespacelock = user_namespacelock_full(userid, index_locktype);
        }
        free(userid);
    }

    r = mboxname_lock(legacy_dirs ? mailbox_name(mailbox) : mailbox_uniqueid(mailbox), &listitem->l, locktype);
    if (r) return r;

    return r;
}


static int mailbox_reconstruct_append(struct mailbox *mailbox, uint32_t uid, int isarchive,
                                      int has_snoozedannot, int flags)
{
    /* XXX - support archived */
    const char *fname;
    int r = 0;
    struct index_record record;
    struct stat sbuf;
    int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;

    memset(&record, 0, sizeof(struct index_record));

    int remove_temp_spool_file = 0;
    int object_storage_enabled = 0;
#if defined ENABLE_OBJECTSTORE
    object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED);
#endif

    if (isarchive && !object_storage_enabled)
        fname = mailbox_archive_fname(mailbox, uid);
    else
        fname = mailbox_spool_fname(mailbox, uid);

#if defined ENABLE_OBJECTSTORE
    if (object_storage_enabled)
    {
        uint32_t i , count = 0;
        struct message *list = get_list_of_message (mailbox, &count);
        for (i = 0; i < count; i++) {
            if (uid == list[i].message_uid){

                record.uid = uid;
                record.guid = list[i].message_guid;
                remove_temp_spool_file = 1;
                r = objectstore_get(mailbox, &record, fname);
                break;
            }
        }
        discard_list();
    }
#endif

    /* possible if '0.' file exists */
    if (!uid) {
        /* filthy hack - copy the path to '1.' and replace 1 with 0 */
        char *hack;
        if (isarchive && !object_storage_enabled)
            fname = mailbox_archive_fname(mailbox, 1);
        else
            fname = mailbox_spool_fname(mailbox, 1);
        hack = (char *)fname;
        hack[strlen(fname)-2] = '0';
    }

    if (stat(fname, &sbuf) == -1) r = IMAP_MAILBOX_NONEXISTENT;
    else if (sbuf.st_size == 0) r = IMAP_MAILBOX_NONEXISTENT;

    /* no file, nothing to do! */
    if (r) {
        syslog(LOG_ERR, "%s uid %u not found", mailbox_name(mailbox), uid);
        printf("%s uid %u not found", mailbox_name(mailbox), uid);
        r = 0;

        if (make_changes)
            unlink(fname);

        goto out;
    }

    r = message_parse(fname, &record);
    if (r) goto out;

    if (isarchive)
        record.internal_flags |= FLAG_INTERNAL_ARCHIVED;

    if (has_snoozedannot)
        record.internal_flags |= FLAG_INTERNAL_SNOOZED;

    /* copy the timestamp from the file if not calculated */
    if (!record.internaldate)
        record.internaldate = sbuf.st_mtime;

    if (uid > mailbox->i.last_uid) {
        printf("%s uid %u found - adding\n", mailbox_name(mailbox), uid);
        syslog(LOG_ERR, "%s uid %u found - adding", mailbox_name(mailbox), uid);
        record.uid = uid;
    }
    else {
        char *oldfname;
        char *newfname;

        printf("%s uid %u rediscovered - appending\n", mailbox_name(mailbox), uid);
        syslog(LOG_ERR, "%s uid %u rediscovered - appending", mailbox_name(mailbox), uid);
        /* XXX - check firstexpunged? */
        record.uid = mailbox->i.last_uid + 1;

        if (!make_changes) {
            r = 0 ;
            goto out;
        }

        oldfname = xstrdup(fname);
        newfname = xstrdup(mailbox_record_fname(mailbox, &record));
        r = rename(oldfname, newfname);
        free(oldfname);
        free(newfname);
        if (r) {
            r = IMAP_IOERROR;
            goto out ;
        }
    }

    /* XXX - inform of changes */
    if (!make_changes){
        r = 0 ;
        goto out;
    }

    /* always allow reconstruct to add records, even if we'd hit GUID limits */
    record.ignorelimits = 1;

    r = mailbox_append_index_record(mailbox, &record);

    /* XXX - copy per-message annotations? */

out:
    if (remove_temp_spool_file)
        remove(fname);
    return r;
}


static int mailbox_update_webdav(struct mailbox *mailbox,
                                 const struct index_record *new)
{
    struct webdav_db *webdavdb = NULL;
    struct param *param;
    struct body *body = NULL;
    struct webdav_data *wdata = NULL;
    struct buf resource = BUF_INITIALIZER;
    int r = 0;

    r = mailbox_cacherecord(mailbox, new);
    if (r) goto done;

    /* Get resource URL from filename param in Content-Disposition header */
    message_read_bodystructure(new, &body);
    for (param = body->disposition_params; param; param = param->next) {
        if (!strcmp(param->attribute, "FILENAME")) {
            buf_setcstr(&resource, param->value);
        }
    }
    if (!buf_len(&resource))
        buf_setcstr(&resource, makeuuid());

    webdavdb = mailbox_open_webdav(mailbox);

    /* Find existing record for this resource */
    const mbentry_t mbentry = { .name = (char *)mailbox_name(mailbox),
                                .uniqueid = (char *)mailbox_uniqueid(mailbox) };

    webdav_lookup_resource(webdavdb, &mbentry, buf_cstring(&resource), &wdata, /*tombstones*/1);

    /* if updated by a newer UID, skip - this record doesn't refer to the current item */
    if (wdata->dav.imap_uid > new->uid) goto done;

    if (new->internal_flags & FLAG_INTERNAL_UNLINKED) {
        /* is there an existing record? */
        if (!wdata->dav.imap_uid) goto done;

        /* delete entry */
        r = webdav_delete(webdavdb, wdata->dav.rowid);
    }
    else if (wdata->dav.imap_uid == new->uid) {
        /* just a flags update to an existing record */
        wdata->dav.modseq = new->modseq;
        wdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
        wdata->ref_count *= wdata->dav.alive;
        r = webdav_write(webdavdb, wdata);
    }
    else {
        struct buf msg_buf = BUF_INITIALIZER;
        struct message_guid guid;

        /* Load message containing the resource */
        r = mailbox_map_record(mailbox, new, &msg_buf);
        if (r) goto done;

        /* Calculate GUID for body content only */
        message_guid_generate(&guid, buf_base(&msg_buf) + new->header_size,
                              buf_len(&msg_buf) - new->header_size);
        buf_free(&msg_buf);

        wdata->dav.creationdate = new->internaldate;
        wdata->dav.imap_uid = new->uid;
        wdata->dav.modseq = new->modseq;
        wdata->dav.createdmodseq = new->createdmodseq;
        wdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1;
        wdata->ref_count *= wdata->dav.alive;
        wdata->filename = body->description;
        wdata->type = lcase(body->type);
        wdata->subtype = lcase(body->subtype);
        wdata->res_uid = message_guid_encode(&guid);

        r = webdav_write(webdavdb, wdata);
    }

done:
    buf_free(&resource);
    if (body) {
        message_free_body(body);
        free(body);
    }

    return r;
}


static int _parse_header_data(struct mailbox *mailbox,
                              const char *data, size_t datalen)
{
    if (!datalen) return IMAP_MAILBOX_BADFORMAT;

    struct parseentry_rock rock = { &mailbox->h, BUF_INITIALIZER, 0, 0, 0 };
    int r = dlist_parsesax(data, datalen, 0, parseentry_cb, &rock);

    buf_free(&rock.aclbuf); // should be noop, but cleans up after errors

    return r;
}


static int mxf_read_primer_pack(void *arg, AVIOContext *pb, int tag, int size, UID uid, int64_t klv_offset)
{
    MXFContext *mxf = arg;
    int item_num = avio_rb32(pb);
    int item_len = avio_rb32(pb);

    if (item_len != 18) {
        avpriv_request_sample(pb, "Primer pack item length %d", item_len);
        return AVERROR_PATCHWELCOME;
    }
    if (item_num > 65536 || item_num < 0) {
        av_log(mxf->fc, AV_LOG_ERROR, "item_num %d is too large\n", item_num);
        return AVERROR_INVALIDDATA;
    }
    mxf->local_tags = av_calloc(item_num, item_len);
    if (!mxf->local_tags)
        return AVERROR(ENOMEM);
    mxf->local_tags_count = item_num;
    avio_read(pb, mxf->local_tags, item_num*item_len);
    return 0;
}


static void insert_lock(struct lock **pos, struct lock *lock)
{
    lock->next = *pos;
    *pos = lock;
}


int fuse_fs_utimens(struct fuse_fs *fs, const char *path,
                    const struct timespec tv[2])
{
    fuse_get_context()->private_data = fs->user_data;
    if (fs->op.utimens)
        return fs->op.utimens(path, tv);
    else if(fs->op.utime) {
        struct utimbuf buf;
        buf.actime = tv[0].tv_sec;
        buf.modtime = tv[1].tv_sec;
        return fs->op.utime(path, &buf);
    } else
        return -ENOSYS;
}


int fuse_fs_mkdir(struct fuse_fs *fs, const char *path, mode_t mode)
{
    fuse_get_context()->private_data = fs->user_data;
    if (fs->op.mkdir)
        return fs->op.mkdir(path, mode);
    else
        return -ENOSYS;
}


static int extend_contents(struct fuse_dh *dh, unsigned minsize)
{
    if (minsize > dh->size) {
        char *newptr;
        unsigned newsize = dh->size;
        if (!newsize)
            newsize = 1024;
#ifndef __SOLARIS__
        while (newsize < minsize) {
	    if (newsize >= 0x80000000)
	       	newsize = 0xffffffff;
	    else
	       	newsize *= 2;
        }
#else /* __SOLARIS__ */
        while (newsize < minsize)
            newsize *= 2;
#endif /* __SOLARIS__ */

        newptr = (char *) realloc(dh->contents, newsize);
        if (!newptr) {
            dh->error = -ENOMEM;
            return -1;
        }
        dh->contents = newptr;
        dh->size = newsize;
    }
    return 0;
}


static void delete_lock(struct lock **lockp)
{
    struct lock *l = *lockp;
    *lockp = l->next;
    free(l);
}


static void fuse_restore_intr_signal(int signum)
{
    struct sigaction sa;

    memset(&sa, 0, sizeof(struct sigaction));
    sa.sa_handler = SIG_DFL;
    sigaction(signum, &sa, NULL);
}


static void fuse_lib_flush(fuse_req_t req, fuse_ino_t ino,
                       struct fuse_file_info *fi)
{
    struct fuse *f = req_fuse_prepare(req);
    char *path;
    int err;

    pthread_rwlock_rdlock(&f->tree_lock);
    path = get_path(f, ino);
    if (path && f->conf.debug)
        fprintf(stderr, "FLUSH[%llu]\n", (unsigned long long) fi->fh);
    err = fuse_flush_common(f, req, ino, path, fi);
    free(path);
    pthread_rwlock_unlock(&f->tree_lock);
    reply_err(req, err);
}


static void fuse_lib_write(fuse_req_t req, fuse_ino_t ino, const char *buf,
                       size_t size, off_t off, struct fuse_file_info *fi)
{
    struct fuse *f = req_fuse_prepare(req);
    char *path;
    int res;

    res = -ENOENT;
    pthread_rwlock_rdlock(&f->tree_lock);
    path = get_path(f, ino);
    if (path != NULL) {
        struct fuse_intr_data d;
        if (f->conf.debug)
            fprintf(stderr, "WRITE%s[%llu] %lu bytes to %llu\n",
                    fi->writepage ? "PAGE" : "", (unsigned long long) fi->fh,
                    (unsigned long) size, (unsigned long long) off);

        fuse_prepare_interrupt(f, req, &d);
        res = fuse_fs_write(f->fs, path, buf, size, off, fi);
        fuse_finish_interrupt(f, req, &d);
        free(path);
    }
    pthread_rwlock_unlock(&f->tree_lock);

    if (res >= 0) {
        if (f->conf.debug)
            fprintf(stderr, "   WRITE%s[%llu] %u bytes\n",
                    fi->writepage ? "PAGE" : "", (unsigned long long) fi->fh,
                    res);
        if ((size_t) res > size)
            fprintf(stderr, "fuse: wrote too many bytes");
        fuse_reply_write(req, res);
    } else
        reply_err(req, res);
}


static void fuse_do_release(struct fuse *f, fuse_ino_t ino, const char *path,
                            struct fuse_file_info *fi)
{
    struct node *node;
    int unlink_hidden = 0;

    fuse_fs_release(f->fs, path ? path : "-", fi);

    pthread_mutex_lock(&f->lock);
    node = get_node(f, ino);
    assert(node->open_count > 0);
    --node->open_count;
    if (node->is_hidden && !node->open_count) {
        unlink_hidden = 1;
        node->is_hidden = 0;
    }
    pthread_mutex_unlock(&f->lock);

    if(unlink_hidden && path)
        fuse_fs_unlink(f->fs, path);
}


static int mtime_eq(const struct stat *stbuf, const struct timespec *ts)
{
    return stbuf->st_mtime == ts->tv_sec && ST_MTIM_NSEC(stbuf) == ts->tv_nsec;
}


static void fuse_lib_getattr(fuse_req_t req, fuse_ino_t ino,
                             struct fuse_file_info *fi)
{
    struct fuse *f = req_fuse_prepare(req);
    struct stat buf;
    char *path;
    int err;

    (void) fi;
    memset(&buf, 0, sizeof(buf));

    err = -ENOENT;
    pthread_rwlock_rdlock(&f->tree_lock);
    path = get_path(f, ino);
    if (path != NULL) {
        struct fuse_intr_data d;
        fuse_prepare_interrupt(f, req, &d);
        err = fuse_fs_getattr(f->fs, path, &buf);
        fuse_finish_interrupt(f, req, &d);
        free(path);
    }
    pthread_rwlock_unlock(&f->tree_lock);
    if (!err) {
#ifdef __SOLARIS__
        if (f->conf.auto_cache) {
            pthread_mutex_lock(&f->lock);
            update_stat(get_node(f, ino), &buf);
            pthread_mutex_unlock(&f->lock);
        }
#endif /* __SOLARIS__ */
        set_stat(f, ino, &buf);
        fuse_reply_attr(req, &buf, f->conf.attr_timeout);
    } else
        reply_err(req, err);
}


static void fuse_lib_unlink(fuse_req_t req, fuse_ino_t parent,
                            const char *name)
{
    struct fuse *f = req_fuse_prepare(req);
    char *path;
    int err;

    err = -ENOENT;
    pthread_rwlock_wrlock(&f->tree_lock);
    path = get_path_name(f, parent, name);
    if (path != NULL) {
        struct fuse_intr_data d;
        if (f->conf.debug)
            fprintf(stderr, "UNLINK %s\n", path);
        fuse_prepare_interrupt(f, req, &d);
        if (!f->conf.hard_remove && is_open(f, parent, name))
            err = hide_node(f, path, parent, name);
        else {
            err = fuse_fs_unlink(f->fs, path);
            if (!err)
                remove_node(f, parent, name);
        }
        fuse_finish_interrupt(f, req, &d);
        free(path);
    }
    pthread_rwlock_unlock(&f->tree_lock);
    reply_err(req, err);
}


int fuse_is_lib_option(const char *opt)
{
    return fuse_lowlevel_is_lib_option(opt) ||
        fuse_opt_match(fuse_lib_opts, opt);
}


int fuse_fs_releasedir(struct fuse_fs *fs, const char *path,
                       struct fuse_file_info *fi)
{
    fuse_get_context()->private_data = fs->user_data;
    if (fs->op.releasedir)
        return fs->op.releasedir(path, fi);
    else
        return 0;
}


int fuse_fs_bmap(struct fuse_fs *fs, const char *path, size_t blocksize,
                 uint64_t *idx)
{
    fuse_get_context()->private_data = fs->user_data;
    if (fs->op.bmap)
        return fs->op.bmap(path, blocksize, idx);
    else
        return -ENOSYS;
}


size_t extract_content_length(char *buffer, size_t size)
{
	char *clen = strstr(buffer, CONTENT_LENGTH);
	size_t len;
	char *end;

	/* Pattern not found */
	if (!clen || clen > buffer + size)
		return SIZE_MAX;

	/* Content-Length extraction */
	len = strtoul(clen + strlen(CONTENT_LENGTH), &end, 10);
	if (*end)
		return SIZE_MAX;

	return len;
}


static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
				    pgoff_t index, pgoff_t end, int whence)
{
	struct page *page;
	struct pagevec pvec;
	pgoff_t indices[PAGEVEC_SIZE];
	bool done = false;
	int i;

	pagevec_init(&pvec, 0);
	pvec.nr = 1;		/* start small: we may be there already */
	while (!done) {
		pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
					pvec.nr, pvec.pages, indices);
		if (!pvec.nr) {
			if (whence == SEEK_DATA)
				index = end;
			break;
		}
		for (i = 0; i < pvec.nr; i++, index++) {
			if (index < indices[i]) {
				if (whence == SEEK_HOLE) {
					done = true;
					break;
				}
				index = indices[i];
			}
			page = pvec.pages[i];
			if (page && !radix_tree_exceptional_entry(page)) {
				if (!PageUptodate(page))
					page = NULL;
			}
			if (index >= end ||
			    (page && whence == SEEK_DATA) ||
			    (!page && whence == SEEK_HOLE)) {
				done = true;
				break;
			}
		}
		shmem_deswap_pagevec(&pvec);
		pagevec_release(&pvec);
		pvec.nr = PAGEVEC_SIZE;
		cond_resched();
	}
	return index;
}


static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);

	buf->f_type = TMPFS_MAGIC;
	buf->f_bsize = PAGE_CACHE_SIZE;
	buf->f_namelen = NAME_MAX;
	if (sbinfo->max_blocks) {
		buf->f_blocks = sbinfo->max_blocks;
		buf->f_bavail =
		buf->f_bfree  = sbinfo->max_blocks -
				percpu_counter_sum(&sbinfo->used_blocks);
	}
	if (sbinfo->max_inodes) {
		buf->f_files = sbinfo->max_inodes;
		buf->f_ffree = sbinfo->free_inodes;
	}
	/* else leave those fields 0 like simple_statfs */
	return 0;
}


static int shmem_writepage(struct page *page, struct writeback_control *wbc)
{
	struct shmem_inode_info *info;
	struct address_space *mapping;
	struct inode *inode;
	swp_entry_t swap;
	pgoff_t index;

	BUG_ON(!PageLocked(page));
	mapping = page->mapping;
	index = page->index;
	inode = mapping->host;
	info = SHMEM_I(inode);
	if (info->flags & VM_LOCKED)
		goto redirty;
	if (!total_swap_pages)
		goto redirty;

	/*
	 * shmem_backing_dev_info's capabilities prevent regular writeback or
	 * sync from ever calling shmem_writepage; but a stacking filesystem
	 * might use ->writepage of its underlying filesystem, in which case
	 * tmpfs should write out to swap only in response to memory pressure,
	 * and not for the writeback threads or sync.
	 */
	if (!wbc->for_reclaim) {
		WARN_ON_ONCE(1);	/* Still happens? Tell us about it! */
		goto redirty;
	}

	/*
	 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
	 * value into swapfile.c, the only way we can correctly account for a
	 * fallocated page arriving here is now to initialize it and write it.
	 *
	 * That's okay for a page already fallocated earlier, but if we have
	 * not yet completed the fallocation, then (a) we want to keep track
	 * of this page in case we have to undo it, and (b) it may not be a
	 * good idea to continue anyway, once we're pushing into swap.  So
	 * reactivate the page, and let shmem_fallocate() quit when too many.
	 */
	if (!PageUptodate(page)) {
		if (inode->i_private) {
			struct shmem_falloc *shmem_falloc;
			spin_lock(&inode->i_lock);
			shmem_falloc = inode->i_private;
			if (shmem_falloc &&
			    index >= shmem_falloc->start &&
			    index < shmem_falloc->next)
				shmem_falloc->nr_unswapped++;
			else
				shmem_falloc = NULL;
			spin_unlock(&inode->i_lock);
			if (shmem_falloc)
				goto redirty;
		}
		clear_highpage(page);
		flush_dcache_page(page);
		SetPageUptodate(page);
	}

	swap = get_swap_page();
	if (!swap.val)
		goto redirty;

	/*
	 * Add inode to shmem_unuse()'s list of swapped-out inodes,
	 * if it's not already there.  Do it now before the page is
	 * moved to swap cache, when its pagelock no longer protects
	 * the inode from eviction.  But don't unlock the mutex until
	 * we've incremented swapped, because shmem_unuse_inode() will
	 * prune a !swapped inode from the swaplist under this mutex.
	 */
	mutex_lock(&shmem_swaplist_mutex);
	if (list_empty(&info->swaplist))
		list_add_tail(&info->swaplist, &shmem_swaplist);

	if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
		swap_shmem_alloc(swap);
		shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));

		spin_lock(&info->lock);
		info->swapped++;
		shmem_recalc_inode(inode);
		spin_unlock(&info->lock);

		mutex_unlock(&shmem_swaplist_mutex);
		BUG_ON(page_mapped(page));
		swap_writepage(page, wbc);
		return 0;
	}

	mutex_unlock(&shmem_swaplist_mutex);
	swapcache_free(swap, NULL);
redirty:
	set_page_dirty(page);
	if (wbc->for_reclaim)
		return AOP_WRITEPAGE_ACTIVATE;	/* Return with page locked */
	unlock_page(page);
	return 0;
}


void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
{
	truncate_inode_pages_range(inode->i_mapping, lstart, lend);
}


static void shmem_destroy_callback(struct rcu_head *head)
{
	struct inode *inode = container_of(head, struct inode, i_rcu);
	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
}


static long shmem_fallocate(struct file *file, int mode, loff_t offset,
							 loff_t len)
{
	struct inode *inode = file->f_path.dentry->d_inode;
	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
	struct shmem_falloc shmem_falloc;
	pgoff_t start, index, end;
	int error;

	mutex_lock(&inode->i_mutex);

	if (mode & FALLOC_FL_PUNCH_HOLE) {
		struct address_space *mapping = file->f_mapping;
		loff_t unmap_start = round_up(offset, PAGE_SIZE);
		loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;

		if ((u64)unmap_end > (u64)unmap_start)
			unmap_mapping_range(mapping, unmap_start,
					    1 + unmap_end - unmap_start, 0);
		shmem_truncate_range(inode, offset, offset + len - 1);
		/* No need to unmap again: hole-punching leaves COWed pages */
		error = 0;
		goto out;
	}

	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
	error = inode_newsize_ok(inode, offset + len);
	if (error)
		goto out;

	start = offset >> PAGE_CACHE_SHIFT;
	end = (offset + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
	/* Try to avoid a swapstorm if len is impossible to satisfy */
	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
		error = -ENOSPC;
		goto out;
	}

	shmem_falloc.start = start;
	shmem_falloc.next  = start;
	shmem_falloc.nr_falloced = 0;
	shmem_falloc.nr_unswapped = 0;
	spin_lock(&inode->i_lock);
	inode->i_private = &shmem_falloc;
	spin_unlock(&inode->i_lock);

	for (index = start; index < end; index++) {
		struct page *page;

		/*
		 * Good, the fallocate(2) manpage permits EINTR: we may have
		 * been interrupted because we are using up too much memory.
		 */
		if (signal_pending(current))
			error = -EINTR;
		else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
			error = -ENOMEM;
		else
			error = shmem_getpage(inode, index, &page, SGP_FALLOC,
									NULL);
		if (error) {
			/* Remove the !PageUptodate pages we added */
			shmem_undo_range(inode,
				(loff_t)start << PAGE_CACHE_SHIFT,
				(loff_t)index << PAGE_CACHE_SHIFT, true);
			goto undone;
		}

		/*
		 * Inform shmem_writepage() how far we have reached.
		 * No need for lock or barrier: we have the page lock.
		 */
		shmem_falloc.next++;
		if (!PageUptodate(page))
			shmem_falloc.nr_falloced++;

		/*
		 * If !PageUptodate, leave it that way so that freeable pages
		 * can be recognized if we need to rollback on error later.
		 * But set_page_dirty so that memory pressure will swap rather
		 * than free the pages we are allocating (and SGP_CACHE pages
		 * might still be clean: we now need to mark those dirty too).
		 */
		set_page_dirty(page);
		unlock_page(page);
		page_cache_release(page);
		cond_resched();
	}

	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
		i_size_write(inode, offset + len);
	inode->i_ctime = CURRENT_TIME;
undone:
	spin_lock(&inode->i_lock);
	inode->i_private = NULL;
	spin_unlock(&inode->i_lock);
out:
	mutex_unlock(&inode->i_mutex);
	return error;
}


int shmem_fill_super(struct super_block *sb, void *data, int silent)
{
	struct inode *inode;
	struct shmem_sb_info *sbinfo;
	int err = -ENOMEM;

	/* Round up to L1_CACHE_BYTES to resist false sharing */
	sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
				L1_CACHE_BYTES), GFP_KERNEL);
	if (!sbinfo)
		return -ENOMEM;

	sbinfo->mode = S_IRWXUGO | S_ISVTX;
	sbinfo->uid = current_fsuid();
	sbinfo->gid = current_fsgid();
	sb->s_fs_info = sbinfo;

#ifdef CONFIG_TMPFS
	/*
	 * Per default we only allow half of the physical ram per
	 * tmpfs instance, limiting inodes to one per page of lowmem;
	 * but the internal instance is left unlimited.
	 */
	if (!(sb->s_flags & MS_NOUSER)) {
		sbinfo->max_blocks = shmem_default_max_blocks();
		sbinfo->max_inodes = shmem_default_max_inodes();
		if (shmem_parse_options(data, sbinfo, false)) {
			err = -EINVAL;
			goto failed;
		}
	}
	sb->s_export_op = &shmem_export_ops;
	sb->s_flags |= MS_NOSEC;
#else
	sb->s_flags |= MS_NOUSER;
#endif

	spin_lock_init(&sbinfo->stat_lock);
	if (percpu_counter_init(&sbinfo->used_blocks, 0))
		goto failed;
	sbinfo->free_inodes = sbinfo->max_inodes;

	sb->s_maxbytes = MAX_LFS_FILESIZE;
	sb->s_blocksize = PAGE_CACHE_SIZE;
	sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
	sb->s_magic = TMPFS_MAGIC;
	sb->s_op = &shmem_ops;
	sb->s_time_gran = 1;
#ifdef CONFIG_TMPFS_XATTR
	sb->s_xattr = shmem_xattr_handlers;
#endif
#ifdef CONFIG_TMPFS_POSIX_ACL
	sb->s_flags |= MS_POSIXACL;
#endif

	inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
	if (!inode)
		goto failed;
	inode->i_uid = sbinfo->uid;
	inode->i_gid = sbinfo->gid;
	sb->s_root = d_make_root(inode);
	if (!sb->s_root)
		goto failed;
	return 0;

failed:
	shmem_put_super(sb);
	return err;
}


static int shmem_replace_page(struct page **pagep, gfp_t gfp,
				struct shmem_inode_info *info, pgoff_t index)
{
	struct page *oldpage, *newpage;
	struct address_space *swap_mapping;
	pgoff_t swap_index;
	int error;

	oldpage = *pagep;
	swap_index = page_private(oldpage);
	swap_mapping = page_mapping(oldpage);

	/*
	 * We have arrived here because our zones are constrained, so don't
	 * limit chance of success by further cpuset and node constraints.
	 */
	gfp &= ~GFP_CONSTRAINT_MASK;
	newpage = shmem_alloc_page(gfp, info, index);
	if (!newpage)
		return -ENOMEM;

	page_cache_get(newpage);
	copy_highpage(newpage, oldpage);
	flush_dcache_page(newpage);

	__set_page_locked(newpage);
	SetPageUptodate(newpage);
	SetPageSwapBacked(newpage);
	set_page_private(newpage, swap_index);
	SetPageSwapCache(newpage);

	/*
	 * Our caller will very soon move newpage out of swapcache, but it's
	 * a nice clean interface for us to replace oldpage by newpage there.
	 */
	spin_lock_irq(&swap_mapping->tree_lock);
	error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
								   newpage);
	if (!error) {
		__inc_zone_page_state(newpage, NR_FILE_PAGES);
		__dec_zone_page_state(oldpage, NR_FILE_PAGES);
	}
	spin_unlock_irq(&swap_mapping->tree_lock);

	if (unlikely(error)) {
		/*
		 * Is this possible?  I think not, now that our callers check
		 * both PageSwapCache and page_private after getting page lock;
		 * but be defensive.  Reverse old to newpage for clear and free.
		 */
		oldpage = newpage;
	} else {
		mem_cgroup_replace_page_cache(oldpage, newpage);
		lru_cache_add_anon(newpage);
		*pagep = newpage;
	}

	ClearPageSwapCache(oldpage);
	set_page_private(oldpage, 0);

	unlock_page(oldpage);
	page_cache_release(oldpage);
	page_cache_release(oldpage);
	return error;
}


static u64 OS_GetSysClockHIGHRES_FULL()
{
	LARGE_INTEGER now;
	QueryPerformanceCounter(&now);
	now.QuadPart -= init_counter.QuadPart;
	return (u64) ((now.QuadPart * 1000000) / frequency.QuadPart);
}


u64 gf_net_get_ntp_ts()
{
	u64 res;
	u32 sec, frac;
	gf_net_get_ntp(&sec, &frac);
	res = sec;
	res<<= 32;
	res |= frac;
	return res;
}


char gf_prompt_get_char()
{
	char ch;
	if (ch_peek != -1) {
		ch = ch_peek;
		ch_peek = -1;
		close_keyboard(1);
		return ch;
	}
	if (0==read(0,&ch,1))
		ch = 0;
	close_keyboard(1);
	return ch;
}


s32 gf_gettimeofday(struct timeval *tp, void *tz) {
	return gettimeofday(tp, tz);
}


u32 gf_sys_get_argc()
{
	return gpac_argc;
}


const char *gf_sys_get_arg(u32 arg)
{
	if (!gpac_argc || !gpac_argv) return NULL;
	if (arg>=gpac_argc) return NULL;
	return gpac_argv[arg];
}


Bool gf_sys_get_rti_os(u32 refresh_time_ms, GF_SystemRTInfo *rti, u32 flags)
{
#if defined(_WIN32_WCE)
	THREADENTRY32 tentry;
	u64 total_cpu_time, process_cpu_time;
	DWORD orig_perm;
#endif
	MEMORYSTATUS ms;
	u64 creation, exit, kernel, user, process_k_u_time, proc_idle_time, proc_k_u_time;
	u32 entry_time;
	HANDLE hSnapShot;

	assert(sys_init);

	if (!rti) return GF_FALSE;

	proc_idle_time = proc_k_u_time = process_k_u_time = 0;

	entry_time = gf_sys_clock();
	if (last_update_time && (entry_time - last_update_time < refresh_time_ms)) {
		memcpy(rti, &the_rti, sizeof(GF_SystemRTInfo));
		return GF_FALSE;
	}

	if (flags & GF_RTI_SYSTEM_MEMORY_ONLY) {
		memset(rti, 0, sizeof(GF_SystemRTInfo));
		rti->sampling_instant = last_update_time;
		GlobalMemoryStatus(&ms);
		rti->physical_memory = ms.dwTotalPhys;
		rti->physical_memory_avail = ms.dwAvailPhys;
#ifdef GPAC_MEMORY_TRACKING
		rti->gpac_memory = (u64) gpac_allocated_memory;
#endif
		return GF_TRUE;
	}

#if defined (_WIN32_WCE)

	total_cpu_time = process_cpu_time = 0;

	/*get a snapshot of all running threads*/
	orig_perm = GetCurrentPermissions();
	SetProcPermissions(0xFFFFFFFF);
	hSnapShot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
	if (hSnapShot) {
		tentry.dwSize = sizeof(THREADENTRY32);
		the_rti.thread_count = 0;
		/*note we always act as if GF_RTI_ALL_PROCESSES_TIMES flag is set, since there is no other way
		to enumerate threads from a process, and GetProcessTimes doesn't exist on CE*/
		if (Thread32First(hSnapShot, &tentry)) {
			do {
				/*get thread times*/
				if (GetThreadTimes( (HANDLE) tentry.th32ThreadID, (FILETIME *) &creation, (FILETIME *) &exit, (FILETIME *) &kernel, (FILETIME *) &user)) {
					total_cpu_time += user + kernel;
					if (tentry.th32OwnerProcessID==the_rti.pid) {
						process_cpu_time += user + kernel;
						the_rti.thread_count ++;
					}
				}
			} while (Thread32Next(hSnapShot, &tentry));
		}
		CloseToolhelp32Snapshot(hSnapShot);
	}

	if (flags & GF_RTI_PROCESS_MEMORY) {
		HEAPLIST32 hlentry;
		HEAPENTRY32 hentry;
		the_rti.process_memory = 0;
		hlentry.dwSize = sizeof(HEAPLIST32);
		hSnapShot = CreateToolhelp32Snapshot(TH32CS_SNAPHEAPLIST, the_rti.pid);
		if (hSnapShot && Heap32ListFirst(hSnapShot, &hlentry)) {
			do {
				hentry.dwSize = sizeof(hentry);
				if (Heap32First(hSnapShot, &hentry, hlentry.th32ProcessID, hlentry.th32HeapID)) {
					do {
						the_rti.process_memory += hentry.dwBlockSize;
					} while (Heap32Next(hSnapShot, &hentry));
				}
			} while (Heap32ListNext(hSnapShot, &hlentry));
		}
		CloseToolhelp32Snapshot(hSnapShot);
	}
	SetProcPermissions(orig_perm);
	total_cpu_time /= 10;
	process_cpu_time /= 10;

#else
	/*XP-SP1 and Win2003 servers only have GetSystemTimes support. This will give a better estimation
	of CPU usage since we can take into account the idle time*/
	if (MyGetSystemTimes) {
		u64 u_time;
		MyGetSystemTimes(&proc_idle_time, &proc_k_u_time, &u_time);
		proc_k_u_time += u_time;
		proc_idle_time /= 10;
		proc_k_u_time /= 10;
	}
	/*same rq for NtQuerySystemInformation*/
	else if (MyQuerySystemInfo) {
		DWORD ret;
		SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION info;
		MyQuerySystemInfo(0x8 /*SystemProcessorPerformanceInformation*/, &info, sizeof(info), &ret);
		if (ret && (ret<=sizeof(info))) {
			proc_idle_time = info.IdleTime.QuadPart / 10;
			proc_k_u_time = (info.KernelTime.QuadPart + info.UserTime.QuadPart) / 10;
		}
	}
	/*no special API available, ONLY FETCH TIMES if requested (may eat up some time)*/
	else if (flags & GF_RTI_ALL_PROCESSES_TIMES) {
		PROCESSENTRY32 pentry;
		/*get a snapshot of all running threads*/
		hSnapShot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
		if (!hSnapShot) return GF_FALSE;
		pentry.dwSize = sizeof(PROCESSENTRY32);
		if (Process32First(hSnapShot, &pentry)) {
			do {
				HANDLE procH = NULL;
				if (pentry.th32ProcessID) procH = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pentry.th32ProcessID);
				if (procH && GetProcessTimes(procH, (FILETIME *) &creation, (FILETIME *) &exit, (FILETIME *) &kernel, (FILETIME *) &user) ) {
					user += kernel;
					proc_k_u_time += user;
					if (pentry.th32ProcessID==the_rti.pid) {
						process_k_u_time = user;
						//nb_threads = pentry.cntThreads;
					}
				}
				if (procH) CloseHandle(procH);
			} while (Process32Next(hSnapShot, &pentry));
		}
		CloseHandle(hSnapShot);
		proc_k_u_time /= 10;
	}

	if (!process_k_u_time) {
		HANDLE procH = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, the_rti.pid);
		if (procH && GetProcessTimes(procH, (FILETIME *) &creation, (FILETIME *) &exit, (FILETIME *) &kernel, (FILETIME *) &user) ) {
			process_k_u_time = user + kernel;
		}
		if (procH) CloseHandle(procH);
		if (!process_k_u_time) return GF_FALSE;
	}
	process_k_u_time /= 10;

	/*this won't cost a lot*/
	if (MyGetProcessMemoryInfo) {
		PROCESS_MEMORY_COUNTERS pmc;
		HANDLE procH = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, the_rti.pid);
		MyGetProcessMemoryInfo(procH, &pmc, sizeof (pmc));
		the_rti.process_memory = pmc.WorkingSetSize;
		if (procH) CloseHandle(procH);
	}
	/*THIS IS VERY HEAVY (eats up mem and time) - only perform if requested*/
	else if (flags & GF_RTI_PROCESS_MEMORY) {
		HEAPLIST32 hlentry;
		HEAPENTRY32 hentry;
		the_rti.process_memory = 0;
		hlentry.dwSize = sizeof(HEAPLIST32);
		hSnapShot = CreateToolhelp32Snapshot(TH32CS_SNAPHEAPLIST, the_rti.pid);
		if (hSnapShot && Heap32ListFirst(hSnapShot, &hlentry)) {
			do {
				hentry.dwSize = sizeof(hentry);
				if (Heap32First(&hentry, hlentry.th32ProcessID, hlentry.th32HeapID)) {
					do {
						the_rti.process_memory += hentry.dwBlockSize;
					} while (Heap32Next(&hentry));
				}
			} while (Heap32ListNext(hSnapShot, &hlentry));
		}
		CloseHandle(hSnapShot);
	}
#endif

	the_rti.sampling_instant = last_update_time;

	if (last_update_time) {
		the_rti.sampling_period_duration = entry_time - last_update_time;
		the_rti.process_cpu_time_diff = (u32) ((process_k_u_time - last_process_k_u_time)/1000);

#if defined(_WIN32_WCE)
		the_rti.total_cpu_time_diff = (u32) ((total_cpu_time - last_total_k_u_time)/1000);
		/*we're not that accurate....*/
		if (the_rti.total_cpu_time_diff > the_rti.sampling_period_duration)
			the_rti.sampling_period_duration = the_rti.total_cpu_time_diff;

		/*rough values*/
		the_rti.cpu_idle_time = the_rti.sampling_period_duration - the_rti.total_cpu_time_diff;
		if (!the_rti.sampling_period_duration) the_rti.sampling_period_duration=1;
		the_rti.total_cpu_usage = (u32) (100 * the_rti.total_cpu_time_diff / the_rti.sampling_period_duration);
		if (the_rti.total_cpu_time_diff + the_rti.cpu_idle_time==0) the_rti.total_cpu_time_diff ++;
		the_rti.process_cpu_usage = (u32) (100*the_rti.process_cpu_time_diff / (the_rti.total_cpu_time_diff + the_rti.cpu_idle_time) );

#else
		/*oops, we have no choice but to assume 100% cpu usage during this period*/
		if (!proc_k_u_time) {
			the_rti.total_cpu_time_diff = the_rti.sampling_period_duration;
			proc_k_u_time = last_proc_k_u_time + the_rti.sampling_period_duration;
			the_rti.cpu_idle_time = 0;
			the_rti.total_cpu_usage = 100;
			if (the_rti.sampling_period_duration)
				the_rti.process_cpu_usage = (u32) (100*the_rti.process_cpu_time_diff / the_rti.sampling_period_duration);
		} else {
			u64 samp_sys_time, idle;
			the_rti.total_cpu_time_diff = (u32) ((proc_k_u_time - last_proc_k_u_time)/1000);

			/*we're not that accurate....*/
			if (the_rti.total_cpu_time_diff > the_rti.sampling_period_duration) {
				the_rti.sampling_period_duration = the_rti.total_cpu_time_diff;
			}

			if (!proc_idle_time)
				proc_idle_time = last_proc_idle_time + (the_rti.sampling_period_duration - the_rti.total_cpu_time_diff);

			samp_sys_time = proc_k_u_time - last_proc_k_u_time;
			idle = proc_idle_time - last_proc_idle_time;
			the_rti.cpu_idle_time = (u32) (idle/1000);
			if (samp_sys_time) {
				the_rti.total_cpu_usage = (u32) ( (samp_sys_time - idle) / (samp_sys_time / 100) );
				the_rti.process_cpu_usage = (u32) (100*the_rti.process_cpu_time_diff / (samp_sys_time/1000));
			}
		}
#endif
	}
	last_update_time = entry_time;
	last_process_k_u_time = process_k_u_time;

	GlobalMemoryStatus(&ms);
	the_rti.physical_memory = ms.dwTotalPhys;
#ifdef GPAC_MEMORY_TRACKING
	the_rti.gpac_memory = (u64) gpac_allocated_memory;
#endif
	the_rti.physical_memory_avail = ms.dwAvailPhys;

#if defined(_WIN32_WCE)
	last_total_k_u_time = total_cpu_time;
	if (!the_rti.process_memory) the_rti.process_memory = mem_usage_at_startup - ms.dwAvailPhys;
#else
	last_proc_idle_time = proc_idle_time;
	last_proc_k_u_time = proc_k_u_time;
#endif

	if (!the_rti.gpac_memory) the_rti.gpac_memory = the_rti.process_memory;

	memcpy(rti, &the_rti, sizeof(GF_SystemRTInfo));
	return GF_TRUE;
}


void gf_isom_ismacryp_delete_sample(GF_ISMASample *samp)
{
	if (!samp) return;
	if (samp->data && samp->dataLength) gf_free(samp->data);
	if (samp->key_indicator) gf_free(samp->key_indicator);
	gf_free(samp);
}


void gf_isom_cenc_set_saiz_saio(GF_SampleEncryptionBox *senc, GF_SampleTableBox *stbl, GF_TrackFragmentBox  *traf, u32 len, Bool saio_32bits, Bool use_multikey)
{
	u32  i;
	GF_List **child_boxes = stbl ? &stbl->child_boxes : &traf->child_boxes;
	if (!senc->cenc_saiz) {
		senc->cenc_saiz = (GF_SampleAuxiliaryInfoSizeBox *) gf_isom_box_new_parent(child_boxes, GF_ISOM_BOX_TYPE_SAIZ);
		if (!senc->cenc_saiz) return;
		//as per 3rd edition of cenc "so content SHOULD be created omitting these optional fields" ...
		senc->cenc_saiz->aux_info_type = 0;
		senc->cenc_saiz->aux_info_type_parameter = use_multikey ? 1 : 0;
		if (stbl)
			stbl_on_child_box((GF_Box*)stbl, (GF_Box *)senc->cenc_saiz, GF_FALSE);
		else
			traf_on_child_box((GF_Box*)traf, (GF_Box *)senc->cenc_saiz, GF_FALSE);
	}
	if (!senc->cenc_saio) {
		senc->cenc_saio = (GF_SampleAuxiliaryInfoOffsetBox *) gf_isom_box_new_parent(child_boxes, GF_ISOM_BOX_TYPE_SAIO);
		if (!senc->cenc_saio) return;
		//force using version 1 for saio box, it could be redundant when we use 64 bits for offset
		senc->cenc_saio->version = saio_32bits ? 0 : 1;
		//as per 3rd edition of cenc "so content SHOULD be created omitting these optional fields" ...
		senc->cenc_saio->aux_info_type = 0;
		senc->cenc_saiz->aux_info_type_parameter = use_multikey ? 1 : 0;
		senc->cenc_saio->entry_count = 1;
		if (stbl)
			stbl_on_child_box((GF_Box*)stbl, (GF_Box *)senc->cenc_saio, GF_FALSE);
		else
			traf_on_child_box((GF_Box*)traf, (GF_Box *)senc->cenc_saio, GF_FALSE);
	}

	if (!senc->cenc_saiz->sample_count || ((senc->cenc_saiz->default_sample_info_size==len) && len) ) {
		senc->cenc_saiz->sample_count ++;
		senc->cenc_saiz->default_sample_info_size = len;
	} else {
		if (senc->cenc_saiz->sample_count + 1 > senc->cenc_saiz->sample_alloc) {
			if (!senc->cenc_saiz->sample_alloc) senc->cenc_saiz->sample_alloc = senc->cenc_saiz->sample_count+1;
			else senc->cenc_saiz->sample_alloc *= 2;

			senc->cenc_saiz->sample_info_size = (u8*)gf_realloc(senc->cenc_saiz->sample_info_size, sizeof(u8)*(senc->cenc_saiz->sample_alloc));
		}

		if (senc->cenc_saiz->default_sample_info_size || (senc->cenc_saiz->sample_count==1)) {
			for (i=0; i<senc->cenc_saiz->sample_count; i++)
				senc->cenc_saiz->sample_info_size[i] = senc->cenc_saiz->default_sample_info_size;
			senc->cenc_saiz->default_sample_info_size = 0;
		}
		senc->cenc_saiz->sample_info_size[senc->cenc_saiz->sample_count] = len;
		senc->cenc_saiz->sample_count++;
	}
}


GF_Err gf_isom_set_adobe_protection(GF_ISOFile *the_file, u32 trackNumber, u32 desc_index, u32 scheme_type, u32 scheme_version, Bool is_selective_enc, char *metadata, u32 len)
{
	GF_ProtectionSchemeInfoBox *sinf;

	//setup generic protection
#ifndef GPAC_DISABLE_ISOM_WRITE
	GF_Err e;
	e = isom_set_protected_entry(the_file, trackNumber, desc_index, 1, 0, scheme_type, scheme_version, NULL, GF_FALSE, &sinf);
	if (e) return e;
#else
	return GF_NOT_SUPPORTED;
#endif

	sinf->info->adkm = (GF_AdobeDRMKeyManagementSystemBox *)gf_isom_box_new_parent(&sinf->info->child_boxes, GF_ISOM_BOX_TYPE_ADKM);
	if (!sinf->info->adkm) return GF_OUT_OF_MEM;

	sinf->info->adkm->header = (GF_AdobeDRMHeaderBox *)gf_isom_box_new_parent(&sinf->info->adkm->child_boxes, GF_ISOM_BOX_TYPE_AHDR);
	if (!sinf->info->adkm->header) return GF_OUT_OF_MEM;

	sinf->info->adkm->header->std_enc_params = (GF_AdobeStdEncryptionParamsBox *)gf_isom_box_new_parent(& sinf->info->adkm->header->child_boxes, GF_ISOM_BOX_TYPE_APRM);
	if (!sinf->info->adkm->header->std_enc_params) return GF_OUT_OF_MEM;

	sinf->info->adkm->header->std_enc_params->enc_info = (GF_AdobeEncryptionInfoBox *)gf_isom_box_new_parent(&sinf->info->adkm->header->std_enc_params->child_boxes, GF_ISOM_BOX_TYPE_AEIB);
	if (!sinf->info->adkm->header->std_enc_params->enc_info) return GF_OUT_OF_MEM;

	sinf->info->adkm->header->std_enc_params->enc_info->enc_algo = (char *)gf_malloc(8*sizeof(char));
	if (!sinf->info->adkm->header->std_enc_params->enc_info->enc_algo) return GF_OUT_OF_MEM;

	strcpy(sinf->info->adkm->header->std_enc_params->enc_info->enc_algo, "AES-CBC");
	sinf->info->adkm->header->std_enc_params->enc_info->key_length = 16;

	sinf->info->adkm->header->std_enc_params->key_info = (GF_AdobeKeyInfoBox *)gf_isom_box_new_parent(&sinf->info->adkm->header->std_enc_params->child_boxes, GF_ISOM_BOX_TYPE_AKEY);
	if (!sinf->info->adkm->header->std_enc_params->key_info) return GF_OUT_OF_MEM;

	sinf->info->adkm->header->std_enc_params->key_info->params = (GF_AdobeFlashAccessParamsBox *)gf_isom_box_new_parent(&sinf->info->adkm->header->std_enc_params->key_info->child_boxes, GF_ISOM_BOX_TYPE_FLXS);
	if (!sinf->info->adkm->header->std_enc_params->key_info->params) return GF_OUT_OF_MEM;

	if (metadata && len) {
		sinf->info->adkm->header->std_enc_params->key_info->params->metadata = (char *)gf_malloc((len+1)*sizeof(char));
		if (!sinf->info->adkm->header->std_enc_params->key_info->params->metadata) return GF_OUT_OF_MEM;

		strncpy(sinf->info->adkm->header->std_enc_params->key_info->params->metadata, metadata, len);
		sinf->info->adkm->header->std_enc_params->key_info->params->metadata[len] = 0;
	}

	sinf->info->adkm->au_format = (GF_AdobeDRMAUFormatBox *)gf_isom_box_new_parent(&sinf->info->adkm->child_boxes, GF_ISOM_BOX_TYPE_ADAF);
	if (!sinf->info->adkm->au_format) return GF_OUT_OF_MEM;

	sinf->info->adkm->au_format->selective_enc = is_selective_enc ? 0x10 : 0x00;
	sinf->info->adkm->au_format->IV_length = 16;

	return GF_OK;
}


GF_Err gf_isom_get_ismacryp_info(GF_ISOFile *the_file, u32 trackNumber, u32 sampleDescriptionIndex, u32 *outOriginalFormat, u32 *outSchemeType, u32 *outSchemeVersion, const char **outSchemeURI, const char **outKMS_URI, Bool *outSelectiveEncryption, u32 *outIVLength, u32 *outKeyIndicationLength)
{
	GF_TrackBox *trak;
	GF_ProtectionSchemeInfoBox *sinf;

	trak = gf_isom_get_track_from_file(the_file, trackNumber);
	if (!trak) return GF_BAD_PARAM;

	sinf = isom_get_sinf_entry(trak, sampleDescriptionIndex, GF_ISOM_ISMACRYP_SCHEME, NULL);
	if (!sinf) return GF_OK;

	if (outOriginalFormat) {
		*outOriginalFormat = sinf->original_format->data_format;
		if (IsMP4Description(sinf->original_format->data_format)) *outOriginalFormat = GF_ISOM_SUBTYPE_MPEG4;
	}
	if (outSchemeType) *outSchemeType = sinf->scheme_type->scheme_type;
	if (outSchemeVersion) *outSchemeVersion = sinf->scheme_type->scheme_version;
	if (outSchemeURI) *outSchemeURI = sinf->scheme_type->URI;

	if (sinf->info && sinf->info->ikms) {
		if (outKMS_URI) *outKMS_URI = sinf->info->ikms->URI;
	} else {
		if (outKMS_URI) *outKMS_URI = NULL;
	}
	if (sinf->info && sinf->info->isfm) {
		if (outSelectiveEncryption) *outSelectiveEncryption = sinf->info->isfm->selective_encryption;
		if (outIVLength) *outIVLength = sinf->info->isfm->IV_length;
		if (outKeyIndicationLength) *outKeyIndicationLength = sinf->info->isfm->key_indicator_length;
	} else {
		if (outSelectiveEncryption) *outSelectiveEncryption = GF_FALSE;
		if (outIVLength) *outIVLength = 0;
		if (outKeyIndicationLength) *outKeyIndicationLength = 0;
	}
	return GF_OK;
}


static NTSTATUS kdc_task_init(struct task_server *task)
{
	struct kdc_server *kdc;
	NTSTATUS status;
	struct interface *ifaces;

	switch (lpcfg_server_role(task->lp_ctx)) {
	case ROLE_STANDALONE:
		task_server_terminate(task, "kdc: no KDC required in standalone configuration", false);
		return NT_STATUS_INVALID_DOMAIN_ROLE;
	case ROLE_DOMAIN_MEMBER:
		task_server_terminate(task, "kdc: no KDC required in member server configuration", false);
		return NT_STATUS_INVALID_DOMAIN_ROLE;
	case ROLE_DOMAIN_PDC:
	case ROLE_DOMAIN_BDC:
	case ROLE_IPA_DC:
		task_server_terminate(
		    task, "Cannot start KDC as a 'classic Samba' DC", false);
		return NT_STATUS_INVALID_DOMAIN_ROLE;
	case ROLE_ACTIVE_DIRECTORY_DC:
		/* Yes, we want a KDC */
		break;
	}

	load_interface_list(task, task->lp_ctx, &ifaces);

	if (iface_list_count(ifaces) == 0) {
		task_server_terminate(task, "kdc: no network interfaces configured", false);
		return NT_STATUS_UNSUCCESSFUL;
	}

	task_server_set_title(task, "task[kdc]");

	kdc = talloc_zero(task, struct kdc_server);
	if (kdc == NULL) {
		task_server_terminate(task, "kdc: out of memory", true);
		return NT_STATUS_NO_MEMORY;
	}

	kdc->task = task;
	task->private_data = kdc;

	/* start listening on the configured network interfaces */
	status = kdc_startup_interfaces(kdc, task->lp_ctx, ifaces,
					task->model_ops);
	if (!NT_STATUS_IS_OK(status)) {
		task_server_terminate(task, "kdc failed to setup interfaces", true);
		return status;
	}

	return NT_STATUS_OK;
}


static NTSTATUS can_rename(connection_struct *conn, files_struct *fsp,
			uint16_t dirtype)
{
	if (!CAN_WRITE(conn)) {
		return NT_STATUS_MEDIA_WRITE_PROTECTED;
	}

	if ((dirtype & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) !=
			(FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) {
		/* Only bother to read the DOS attribute if we might deny the
		   rename on the grounds of attribute mismatch. */
		uint32_t fmode = fdos_mode(fsp);
		if ((fmode & ~dirtype) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) {
			return NT_STATUS_NO_SUCH_FILE;
		}
	}

	if (S_ISDIR(fsp->fsp_name->st.st_ex_mode)) {
		if (fsp->posix_flags & FSP_POSIX_FLAGS_RENAME) {
			return NT_STATUS_OK;
		}

		/* If no pathnames are open below this
		   directory, allow the rename. */

		if (lp_strict_rename(SNUM(conn))) {
			/*
			 * Strict rename, check open file db.
			 */
			if (have_file_open_below(fsp->conn, fsp->fsp_name)) {
				return NT_STATUS_ACCESS_DENIED;
			}
		} else if (file_find_subpath(fsp)) {
			/*
			 * No strict rename, just look in local process.
			 */
			return NT_STATUS_ACCESS_DENIED;
		}
		return NT_STATUS_OK;
	}

	if (fsp->access_mask & (DELETE_ACCESS|FILE_WRITE_ATTRIBUTES)) {
		return NT_STATUS_OK;
	}

	return NT_STATUS_ACCESS_DENIED;
}


static NTSTATUS parent_dirname_compatible_open(connection_struct *conn,
					const struct smb_filename *smb_fname_dst_in)
{
	struct smb_filename *smb_fname_parent = NULL;
	struct file_id id;
	files_struct *fsp = NULL;
	int ret;
	NTSTATUS status;

	status = SMB_VFS_PARENT_PATHNAME(conn,
					 talloc_tos(),
					 smb_fname_dst_in,
					 &smb_fname_parent,
					 NULL);
	if (!NT_STATUS_IS_OK(status)) {
		return status;
	}

	ret = vfs_stat(conn, smb_fname_parent);
	if (ret == -1) {
		return map_nt_error_from_unix(errno);
	}

	/*
	 * We're only checking on this smbd here, mostly good
	 * enough.. and will pass tests.
	 */

	id = vfs_file_id_from_sbuf(conn, &smb_fname_parent->st);
	for (fsp = file_find_di_first(conn->sconn, id, true); fsp;
			fsp = file_find_di_next(fsp, true)) {
		if (fsp->access_mask & DELETE_ACCESS) {
			return NT_STATUS_SHARING_VIOLATION;
                }
        }
	return NT_STATUS_OK;
}


NTSTATUS check_path_syntax_posix(char *path)
{
	return check_path_syntax_internal(path, true);
}