summaryrefslogtreecommitdiffstats
path: root/source3/smbd/files.c
blob: efaadffc061cda7569db49ce9186d0af951f76d1 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
/* 
   Unix SMB/CIFS implementation.
   Files[] structure handling
   Copyright (C) Andrew Tridgell 1998
   
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.
   
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "includes.h"
#include "smbd/globals.h"

#define VALID_FNUM(fnum)   (((fnum) >= 0) && ((fnum) < real_max_open_files))

#define FILE_HANDLE_OFFSET 0x1000

/****************************************************************************
 Return a unique number identifying this fsp over the life of this pid.
****************************************************************************/

static unsigned long get_gen_count(void)
{
	if ((++file_gen_counter) == 0)
		return ++file_gen_counter;
	return file_gen_counter;
}

/****************************************************************************
 Find first available file slot.
****************************************************************************/

NTSTATUS file_new(struct smb_request *req, connection_struct *conn,
		  files_struct **result)
{
	int i;
	files_struct *fsp;

	/* we want to give out file handles differently on each new
	   connection because of a common bug in MS clients where they try to
	   reuse a file descriptor from an earlier smb connection. This code
	   increases the chance that the errant client will get an error rather
	   than causing corruption */
	if (first_file == 0) {
		first_file = (sys_getpid() ^ (int)time(NULL)) % real_max_open_files;
	}

	/* TODO: Port the id-tree implementation from Samba4 */

	i = bitmap_find(file_bmap, first_file);
	if (i == -1) {
		DEBUG(0,("ERROR! Out of file structures\n"));
		/* TODO: We have to unconditionally return a DOS error here,
		 * W2k3 even returns ERRDOS/ERRnofids for ntcreate&x with
		 * NTSTATUS negotiated */
		return NT_STATUS_TOO_MANY_OPENED_FILES;
	}

	fsp = SMB_MALLOC_P(files_struct);
	if (!fsp) {
		return NT_STATUS_NO_MEMORY;
	}

	ZERO_STRUCTP(fsp);

	fsp->fh = SMB_MALLOC_P(struct fd_handle);
	if (!fsp->fh) {
		SAFE_FREE(fsp);
		return NT_STATUS_NO_MEMORY;
	}

	ZERO_STRUCTP(fsp->fh);

	fsp->fh->ref_count = 1;
	fsp->fh->fd = -1;

	fsp->conn = conn;
	fsp->fh->gen_id = get_gen_count();
	GetTimeOfDay(&fsp->open_time);

	first_file = (i+1) % real_max_open_files;

	bitmap_set(file_bmap, i);
	files_used++;

	fsp->fnum = i + FILE_HANDLE_OFFSET;
	SMB_ASSERT(fsp->fnum < 65536);

	string_set(&fsp->fsp_name,"");
	
	DLIST_ADD(Files, fsp);

	DEBUG(5,("allocated file structure %d, fnum = %d (%d used)\n",
		 i, fsp->fnum, files_used));

	if (req != NULL) {
		req->chain_fsp = fsp;
	}

	/* A new fsp invalidates the positive and
	  negative fsp_fi_cache as the new fsp is pushed
	  at the start of the list and we search from
	  a cache hit to the *end* of the list. */

	ZERO_STRUCT(fsp_fi_cache);

	conn->num_files_open++;

	*result = fsp;
	return NT_STATUS_OK;
}

/****************************************************************************
 Close all open files for a connection.
****************************************************************************/

void file_close_conn(connection_struct *conn)
{
	files_struct *fsp, *next;
	
	for (fsp=Files;fsp;fsp=next) {
		next = fsp->next;
		if (fsp->conn == conn) {
			close_file(NULL, fsp, SHUTDOWN_CLOSE);
		}
	}
}

/****************************************************************************
 Close all open files for a pid and a vuid.
****************************************************************************/

void file_close_pid(uint16 smbpid, int vuid)
{
	files_struct *fsp, *next;
	
	for (fsp=Files;fsp;fsp=next) {
		next = fsp->next;
		if ((fsp->file_pid == smbpid) && (fsp->vuid == vuid)) {
			close_file(NULL, fsp, SHUTDOWN_CLOSE);
		}
	}
}

/****************************************************************************
 Initialise file structures.
****************************************************************************/

#define MAX_OPEN_FUDGEFACTOR 20

void file_init(void)
{
	int request_max_open_files = lp_max_open_files();
	int real_lim;

	/*
	 * Set the max_open files to be the requested
	 * max plus a fudgefactor to allow for the extra
	 * fd's we need such as log files etc...
	 */
	real_lim = set_maxfiles(request_max_open_files + MAX_OPEN_FUDGEFACTOR);

	real_max_open_files = real_lim - MAX_OPEN_FUDGEFACTOR;

	if (real_max_open_files + FILE_HANDLE_OFFSET + MAX_OPEN_PIPES > 65536)
		real_max_open_files = 65536 - FILE_HANDLE_OFFSET - MAX_OPEN_PIPES;

	if(real_max_open_files != request_max_open_files) {
		DEBUG(1,("file_init: Information only: requested %d \
open files, %d are available.\n", request_max_open_files, real_max_open_files));
	}

	SMB_ASSERT(real_max_open_files > 100);

	file_bmap = bitmap_allocate(real_max_open_files);
	
	if (!file_bmap) {
		exit_server("out of memory in file_init");
	}
}

/****************************************************************************
 Close files open by a specified vuid.
****************************************************************************/

void file_close_user(int vuid)
{
	files_struct *fsp, *next;

	for (fsp=Files;fsp;fsp=next) {
		next=fsp->next;
		if (fsp->vuid == vuid) {
			close_file(NULL, fsp, SHUTDOWN_CLOSE);
		}
	}
}

/****************************************************************************
 Debug to enumerate all open files in the smbd.
****************************************************************************/

void file_dump_open_table(void)
{
	int count=0;
	files_struct *fsp;

	for (fsp=Files;fsp;fsp=fsp->next,count++) {
		DEBUG(10,("Files[%d], fnum = %d, name %s, fd = %d, gen = %lu, fileid=%s\n",
			count, fsp->fnum, fsp->fsp_name, fsp->fh->fd, (unsigned long)fsp->fh->gen_id,
			  file_id_string_tos(&fsp->file_id)));
	}
}

/****************************************************************************
 Find a fsp given a file descriptor.
****************************************************************************/

files_struct *file_find_fd(int fd)
{
	int count=0;
	files_struct *fsp;

	for (fsp=Files;fsp;fsp=fsp->next,count++) {
		if (fsp->fh->fd == fd) {
			if (count > 10) {
				DLIST_PROMOTE(Files, fsp);
			}
			return fsp;
		}
	}

	return NULL;
}

/****************************************************************************
 Find a fsp given a device, inode and file_id.
****************************************************************************/

files_struct *file_find_dif(struct file_id id, unsigned long gen_id)
{
	int count=0;
	files_struct *fsp;

	for (fsp=Files;fsp;fsp=fsp->next,count++) {
		/* We can have a fsp->fh->fd == -1 here as it could be a stat open. */
		if (file_id_equal(&fsp->file_id, &id) &&
		    fsp->fh->gen_id == gen_id ) {
			if (count > 10) {
				DLIST_PROMOTE(Files, fsp);
			}
			/* Paranoia check. */
			if ((fsp->fh->fd == -1) &&
			    (fsp->oplock_type != NO_OPLOCK) &&
			    (fsp->oplock_type != FAKE_LEVEL_II_OPLOCK)) {
				DEBUG(0,("file_find_dif: file %s file_id = %s, gen = %u \
oplock_type = %u is a stat open with oplock type !\n", fsp->fsp_name, 
					 file_id_string_tos(&fsp->file_id),
					 (unsigned int)fsp->fh->gen_id,
					 (unsigned int)fsp->oplock_type ));
				smb_panic("file_find_dif");
			}
			return fsp;
		}
	}

	return NULL;
}

/****************************************************************************
 Check if an fsp still exists.
****************************************************************************/

files_struct *file_find_fsp(files_struct *orig_fsp)
{
	files_struct *fsp;

	for (fsp=Files;fsp;fsp=fsp->next) {
		if (fsp == orig_fsp)
			return fsp;
	}

	return NULL;
}

/****************************************************************************
 Find the first fsp given a device and inode.
 We use a singleton cache here to speed up searching from getfilepathinfo
 calls.
****************************************************************************/

files_struct *file_find_di_first(struct file_id id)
{
	files_struct *fsp;

	if (file_id_equal(&fsp_fi_cache.id, &id)) {
		/* Positive or negative cache hit. */
		return fsp_fi_cache.fsp;
	}

	fsp_fi_cache.id = id;

	for (fsp=Files;fsp;fsp=fsp->next) {
		if (file_id_equal(&fsp->file_id, &id)) {
			/* Setup positive cache. */
			fsp_fi_cache.fsp = fsp;
			return fsp;
		}
	}

	/* Setup negative cache. */
	fsp_fi_cache.fsp = NULL;
	return NULL;
}

/****************************************************************************
 Find the next fsp having the same device and inode.
****************************************************************************/

files_struct *file_find_di_next(files_struct *start_fsp)
{
	files_struct *fsp;

	for (fsp = start_fsp->next;fsp;fsp=fsp->next) {
		if (file_id_equal(&fsp->file_id, &start_fsp->file_id)) {
			return fsp;
		}
	}

	return NULL;
}

/****************************************************************************
 Find a fsp that is open for printing.
****************************************************************************/

files_struct *file_find_print(void)
{
	files_struct *fsp;

	for (fsp=Files;fsp;fsp=fsp->next) {
		if (fsp->print_file) {
			return fsp;
		}
	} 

	return NULL;
}

/****************************************************************************
 Sync open files on a connection.
****************************************************************************/

void file_sync_all(connection_struct *conn)
{
	files_struct *fsp, *next;

	for (fsp=Files;fsp;fsp=next) {
		next=fsp->next;
		if ((conn == fsp->conn) && (fsp->fh->fd != -1)) {
			sync_file(conn, fsp, True /* write through */);
		}
	}
}

/****************************************************************************
 Free up a fsp.
****************************************************************************/

void file_free(struct smb_request *req, files_struct *fsp)
{
	DLIST_REMOVE(Files, fsp);

	string_free(&fsp->fsp_name);

	TALLOC_FREE(fsp->fake_file_handle);

	if (fsp->fh->ref_count == 1) {
		SAFE_FREE(fsp->fh);
	} else {
		fsp->fh->ref_count--;
	}

	if (fsp->notify) {
		notify_remove(fsp->conn->notify_ctx, fsp);
		TALLOC_FREE(fsp->notify);
	}

	/* Ensure this event will never fire. */
	TALLOC_FREE(fsp->oplock_timeout);

	/* Ensure this event will never fire. */
	TALLOC_FREE(fsp->update_write_time_event);

	bitmap_clear(file_bmap, fsp->fnum - FILE_HANDLE_OFFSET);
	files_used--;

	DEBUG(5,("freed files structure %d (%d used)\n",
		 fsp->fnum, files_used));

	fsp->conn->num_files_open--;

	if ((req != NULL) && (fsp == req->chain_fsp)) {
		req->chain_fsp = NULL;
	}

	/* Closing a file can invalidate the positive cache. */
	if (fsp == fsp_fi_cache.fsp) {
		ZERO_STRUCT(fsp_fi_cache);
	}

	/* Drop all remaining extensions. */
	while (fsp->vfs_extension) {
		vfs_remove_fsp_extension(fsp->vfs_extension->owner, fsp);
	}

	/* this is paranoia, just in case someone tries to reuse the
	   information */
	ZERO_STRUCTP(fsp);

	SAFE_FREE(fsp);
}

/****************************************************************************
 Get an fsp from a 16 bit fnum.
****************************************************************************/

files_struct *file_fnum(uint16 fnum)
{
	files_struct *fsp;
	int count=0;

	for (fsp=Files;fsp;fsp=fsp->next, count++) {
		if (fsp->fnum == fnum) {
			if (count > 10) {
				DLIST_PROMOTE(Files, fsp);
			}
			return fsp;
		}
	}
	return NULL;
}

/****************************************************************************
 Get an fsp from a packet given the offset of a 16 bit fnum.
****************************************************************************/

files_struct *file_fsp(struct smb_request *req, uint16 fid)
{
	files_struct *fsp;

	if ((req != NULL) && (req->chain_fsp != NULL)) {
		return req->chain_fsp;
	}

	fsp = file_fnum(fid);
	if ((fsp != NULL) && (req != NULL)) {
		req->chain_fsp = fsp;
	}
	return fsp;
}

/****************************************************************************
 Duplicate the file handle part for a DOS or FCB open.
****************************************************************************/

void dup_file_fsp(struct smb_request *req, files_struct *from,
		      uint32 access_mask, uint32 share_access,
		      uint32 create_options, files_struct *to)
{
	SAFE_FREE(to->fh);

	to->fh = from->fh;
	to->fh->ref_count++;

	to->file_id = from->file_id;
	to->initial_allocation_size = from->initial_allocation_size;
	to->mode = from->mode;
	to->file_pid = from->file_pid;
	to->vuid = from->vuid;
	to->open_time = from->open_time;
	to->access_mask = access_mask;
	to->share_access = share_access;
	to->oplock_type = from->oplock_type;
	to->can_lock = from->can_lock;
	to->can_read = (access_mask & (FILE_READ_DATA)) ? True : False;
	if (!CAN_WRITE(from->conn)) {
		to->can_write = False;
	} else {
		to->can_write = (access_mask & (FILE_WRITE_DATA | FILE_APPEND_DATA)) ? True : False;
	}
	to->print_file = from->print_file;
	to->modified = from->modified;
	to->is_directory = from->is_directory;
	to->aio_write_behind = from->aio_write_behind;
        string_set(&to->fsp_name,from->fsp_name);
}
n1788' href='#n1788'>1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
/*
 * Intel PXA25x and IXP4xx on-chip full speed USB device controllers
 *
 * Copyright (C) 2002 Intrinsyc, Inc. (Frank Becker)
 * Copyright (C) 2003 Robert Schwebel, Pengutronix
 * Copyright (C) 2003 Benedikt Spranger, Pengutronix
 * Copyright (C) 2003 David Brownell
 * Copyright (C) 2003 Joshua Wise
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */

/* #define VERBOSE_DEBUG */

#include <linux/device.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/ioport.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/timer.h>
#include <linux/list.h>
#include <linux/interrupt.h>
#include <linux/mm.h>
#include <linux/platform_device.h>
#include <linux/dma-mapping.h>
#include <linux/irq.h>
#include <linux/clk.h>
#include <linux/err.h>
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/io.h>

#include <asm/byteorder.h>
#include <asm/dma.h>
#include <asm/gpio.h>
#include <asm/system.h>
#include <asm/mach-types.h>
#include <asm/unaligned.h>

#include <linux/usb/ch9.h>
#include <linux/usb/gadget.h>

/*
 * This driver is PXA25x only.  Grab the right register definitions.
 */
#ifdef CONFIG_ARCH_PXA
#include <mach/pxa25x-udc.h>
#endif

#include <asm/mach/udc_pxa2xx.h>


/*
 * This driver handles the USB Device Controller (UDC) in Intel's PXA 25x
 * series processors.  The UDC for the IXP 4xx series is very similar.
 * There are fifteen endpoints, in addition to ep0.
 *
 * Such controller drivers work with a gadget driver.  The gadget driver
 * returns descriptors, implements configuration and data protocols used
 * by the host to interact with this device, and allocates endpoints to
 * the different protocol interfaces.  The controller driver virtualizes
 * usb hardware so that the gadget drivers will be more portable.
 *
 * This UDC hardware wants to implement a bit too much USB protocol, so
 * it constrains the sorts of USB configuration change events that work.
 * The errata for these chips are misleading; some "fixed" bugs from
 * pxa250 a0/a1 b0/b1/b2 sure act like they're still there.
 *
 * Note that the UDC hardware supports DMA (except on IXP) but that's
 * not used here.  IN-DMA (to host) is simple enough, when the data is
 * suitably aligned (16 bytes) ... the network stack doesn't do that,
 * other software can.  OUT-DMA is buggy in most chip versions, as well
 * as poorly designed (data toggle not automatic).  So this driver won't
 * bother using DMA.  (Mostly-working IN-DMA support was available in
 * kernels before 2.6.23, but was never enabled or well tested.)
 */

#define	DRIVER_VERSION	"30-June-2007"
#define	DRIVER_DESC	"PXA 25x USB Device Controller driver"


static const char driver_name [] = "pxa25x_udc";

static const char ep0name [] = "ep0";


#ifdef CONFIG_ARCH_IXP4XX

/* cpu-specific register addresses are compiled in to this code */
#ifdef CONFIG_ARCH_PXA
#error "Can't configure both IXP and PXA"
#endif

/* IXP doesn't yet support <linux/clk.h> */
#define clk_get(dev,name)	NULL
#define clk_enable(clk)		do { } while (0)
#define clk_disable(clk)	do { } while (0)
#define clk_put(clk)		do { } while (0)

#endif

#include "pxa25x_udc.h"


#ifdef	CONFIG_USB_PXA25X_SMALL
#define SIZE_STR	" (small)"
#else
#define SIZE_STR	""
#endif

/* ---------------------------------------------------------------------------
 *	endpoint related parts of the api to the usb controller hardware,
 *	used by gadget driver; and the inner talker-to-hardware core.
 * ---------------------------------------------------------------------------
 */

static void pxa25x_ep_fifo_flush (struct usb_ep *ep);
static void nuke (struct pxa25x_ep *, int status);

/* one GPIO should be used to detect VBUS from the host */
static int is_vbus_present(void)
{
	struct pxa2xx_udc_mach_info		*mach = the_controller->mach;

	if (mach->gpio_vbus) {
		int value = gpio_get_value(mach->gpio_vbus);

		if (mach->gpio_vbus_inverted)
			return !value;
		else
			return !!value;
	}
	if (mach->udc_is_connected)
		return mach->udc_is_connected();
	return 1;
}

/* one GPIO should control a D+ pullup, so host sees this device (or not) */
static void pullup_off(void)
{
	struct pxa2xx_udc_mach_info		*mach = the_controller->mach;
	int off_level = mach->gpio_pullup_inverted;

	if (mach->gpio_pullup)
		gpio_set_value(mach->gpio_pullup, off_level);
	else if (mach->udc_command)
		mach->udc_command(PXA2XX_UDC_CMD_DISCONNECT);
}

static void pullup_on(void)
{
	struct pxa2xx_udc_mach_info		*mach = the_controller->mach;
	int on_level = !mach->gpio_pullup_inverted;

	if (mach->gpio_pullup)
		gpio_set_value(mach->gpio_pullup, on_level);
	else if (mach->udc_command)
		mach->udc_command(PXA2XX_UDC_CMD_CONNECT);
}

static void pio_irq_enable(int bEndpointAddress)
{
        bEndpointAddress &= 0xf;
        if (bEndpointAddress < 8)
                UICR0 &= ~(1 << bEndpointAddress);
        else {
                bEndpointAddress -= 8;
                UICR1 &= ~(1 << bEndpointAddress);
	}
}

static void pio_irq_disable(int bEndpointAddress)
{
        bEndpointAddress &= 0xf;
        if (bEndpointAddress < 8)
                UICR0 |= 1 << bEndpointAddress;
        else {
                bEndpointAddress -= 8;
                UICR1 |= 1 << bEndpointAddress;
        }
}

/* The UDCCR reg contains mask and interrupt status bits,
 * so using '|=' isn't safe as it may ack an interrupt.
 */
#define UDCCR_MASK_BITS         (UDCCR_REM | UDCCR_SRM | UDCCR_UDE)

static inline void udc_set_mask_UDCCR(int mask)
{
	UDCCR = (UDCCR & UDCCR_MASK_BITS) | (mask & UDCCR_MASK_BITS);
}

static inline void udc_clear_mask_UDCCR(int mask)
{
	UDCCR = (UDCCR & UDCCR_MASK_BITS) & ~(mask & UDCCR_MASK_BITS);
}

static inline void udc_ack_int_UDCCR(int mask)
{
	/* udccr contains the bits we dont want to change */
	__u32 udccr = UDCCR & UDCCR_MASK_BITS;

	UDCCR = udccr | (mask & ~UDCCR_MASK_BITS);
}

/*
 * endpoint enable/disable
 *
 * we need to verify the descriptors used to enable endpoints.  since pxa25x
 * endpoint configurations are fixed, and are pretty much always enabled,
 * there's not a lot to manage here.
 *
 * because pxa25x can't selectively initialize bulk (or interrupt) endpoints,
 * (resetting endpoint halt and toggle), SET_INTERFACE is unusable except
 * for a single interface (with only the default altsetting) and for gadget
 * drivers that don't halt endpoints (not reset by set_interface).  that also
 * means that if you use ISO, you must violate the USB spec rule that all
 * iso endpoints must be in non-default altsettings.
 */
static int pxa25x_ep_enable (struct usb_ep *_ep,
		const struct usb_endpoint_descriptor *desc)
{
	struct pxa25x_ep        *ep;
	struct pxa25x_udc       *dev;

	ep = container_of (_ep, struct pxa25x_ep, ep);
	if (!_ep || !desc || ep->desc || _ep->name == ep0name
			|| desc->bDescriptorType != USB_DT_ENDPOINT
			|| ep->bEndpointAddress != desc->bEndpointAddress
			|| ep->fifo_size < le16_to_cpu
						(desc->wMaxPacketSize)) {
		DMSG("%s, bad ep or descriptor\n", __func__);
		return -EINVAL;
	}

	/* xfer types must match, except that interrupt ~= bulk */
	if (ep->bmAttributes != desc->bmAttributes
			&& ep->bmAttributes != USB_ENDPOINT_XFER_BULK
			&& desc->bmAttributes != USB_ENDPOINT_XFER_INT) {
		DMSG("%s, %s type mismatch\n", __func__, _ep->name);
		return -EINVAL;
	}

	/* hardware _could_ do smaller, but driver doesn't */
	if ((desc->bmAttributes == USB_ENDPOINT_XFER_BULK
				&& le16_to_cpu (desc->wMaxPacketSize)
						!= BULK_FIFO_SIZE)
			|| !desc->wMaxPacketSize) {
		DMSG("%s, bad %s maxpacket\n", __func__, _ep->name);
		return -ERANGE;
	}

	dev = ep->dev;
	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN) {
		DMSG("%s, bogus device state\n", __func__);
		return -ESHUTDOWN;
	}

	ep->desc = desc;
	ep->stopped = 0;
	ep->pio_irqs = 0;
	ep->ep.maxpacket = le16_to_cpu (desc->wMaxPacketSize);

	/* flush fifo (mostly for OUT buffers) */
	pxa25x_ep_fifo_flush (_ep);

	/* ... reset halt state too, if we could ... */

	DBG(DBG_VERBOSE, "enabled %s\n", _ep->name);
	return 0;
}

static int pxa25x_ep_disable (struct usb_ep *_ep)
{
	struct pxa25x_ep	*ep;
	unsigned long		flags;

	ep = container_of (_ep, struct pxa25x_ep, ep);
	if (!_ep || !ep->desc) {
		DMSG("%s, %s not enabled\n", __func__,
			_ep ? ep->ep.name : NULL);
		return -EINVAL;
	}
	local_irq_save(flags);

	nuke (ep, -ESHUTDOWN);

	/* flush fifo (mostly for IN buffers) */
	pxa25x_ep_fifo_flush (_ep);

	ep->desc = NULL;
	ep->stopped = 1;

	local_irq_restore(flags);
	DBG(DBG_VERBOSE, "%s disabled\n", _ep->name);
	return 0;
}

/*-------------------------------------------------------------------------*/

/* for the pxa25x, these can just wrap kmalloc/kfree.  gadget drivers
 * must still pass correctly initialized endpoints, since other controller
 * drivers may care about how it's currently set up (dma issues etc).
 */

/*
 *	pxa25x_ep_alloc_request - allocate a request data structure
 */
static struct usb_request *
pxa25x_ep_alloc_request (struct usb_ep *_ep, gfp_t gfp_flags)
{
	struct pxa25x_request *req;

	req = kzalloc(sizeof(*req), gfp_flags);
	if (!req)
		return NULL;

	INIT_LIST_HEAD (&req->queue);
	return &req->req;
}


/*
 *	pxa25x_ep_free_request - deallocate a request data structure
 */
static void
pxa25x_ep_free_request (struct usb_ep *_ep, struct usb_request *_req)
{
	struct pxa25x_request	*req;

	req = container_of (_req, struct pxa25x_request, req);
	WARN_ON(!list_empty (&req->queue));
	kfree(req);
}

/*-------------------------------------------------------------------------*/

/*
 *	done - retire a request; caller blocked irqs
 */
static void done(struct pxa25x_ep *ep, struct pxa25x_request *req, int status)
{
	unsigned		stopped = ep->stopped;

	list_del_init(&req->queue);

	if (likely (req->req.status == -EINPROGRESS))
		req->req.status = status;
	else
		status = req->req.status;

	if (status && status != -ESHUTDOWN)
		DBG(DBG_VERBOSE, "complete %s req %p stat %d len %u/%u\n",
			ep->ep.name, &req->req, status,
			req->req.actual, req->req.length);

	/* don't modify queue heads during completion callback */
	ep->stopped = 1;
	req->req.complete(&ep->ep, &req->req);
	ep->stopped = stopped;
}


static inline void ep0_idle (struct pxa25x_udc *dev)
{
	dev->ep0state = EP0_IDLE;
}

static int
write_packet(volatile u32 *uddr, struct pxa25x_request *req, unsigned max)
{
	u8		*buf;
	unsigned	length, count;

	buf = req->req.buf + req->req.actual;
	prefetch(buf);

	/* how big will this packet be? */
	length = min(req->req.length - req->req.actual, max);
	req->req.actual += length;

	count = length;
	while (likely(count--))
		*uddr = *buf++;

	return length;
}

/*
 * write to an IN endpoint fifo, as many packets as possible.
 * irqs will use this to write the rest later.
 * caller guarantees at least one packet buffer is ready (or a zlp).
 */
static int
write_fifo (struct pxa25x_ep *ep, struct pxa25x_request *req)
{
	unsigned		max;

	max = le16_to_cpu(ep->desc->wMaxPacketSize);
	do {
		unsigned	count;
		int		is_last, is_short;

		count = write_packet(ep->reg_uddr, req, max);

		/* last packet is usually short (or a zlp) */
		if (unlikely (count != max))
			is_last = is_short = 1;
		else {
			if (likely(req->req.length != req->req.actual)
					|| req->req.zero)
				is_last = 0;
			else
				is_last = 1;
			/* interrupt/iso maxpacket may not fill the fifo */
			is_short = unlikely (max < ep->fifo_size);
		}

		DBG(DBG_VERY_NOISY, "wrote %s %d bytes%s%s %d left %p\n",
			ep->ep.name, count,
			is_last ? "/L" : "", is_short ? "/S" : "",
			req->req.length - req->req.actual, req);

		/* let loose that packet. maybe try writing another one,
		 * double buffering might work.  TSP, TPC, and TFS
		 * bit values are the same for all normal IN endpoints.
		 */
		*ep->reg_udccs = UDCCS_BI_TPC;
		if (is_short)
			*ep->reg_udccs = UDCCS_BI_TSP;

		/* requests complete when all IN data is in the FIFO */
		if (is_last) {
			done (ep, req, 0);
			if (list_empty(&ep->queue))
				pio_irq_disable (ep->bEndpointAddress);
			return 1;
		}

		// TODO experiment: how robust can fifo mode tweaking be?
		// double buffering is off in the default fifo mode, which
		// prevents TFS from being set here.

	} while (*ep->reg_udccs & UDCCS_BI_TFS);
	return 0;
}

/* caller asserts req->pending (ep0 irq status nyet cleared); starts
 * ep0 data stage.  these chips want very simple state transitions.
 */
static inline
void ep0start(struct pxa25x_udc *dev, u32 flags, const char *tag)
{
	UDCCS0 = flags|UDCCS0_SA|UDCCS0_OPR;
	USIR0 = USIR0_IR0;
	dev->req_pending = 0;
	DBG(DBG_VERY_NOISY, "%s %s, %02x/%02x\n",
		__func__, tag, UDCCS0, flags);
}

static int
write_ep0_fifo (struct pxa25x_ep *ep, struct pxa25x_request *req)
{
	unsigned	count;
	int		is_short;

	count = write_packet(&UDDR0, req, EP0_FIFO_SIZE);
	ep->dev->stats.write.bytes += count;

	/* last packet "must be" short (or a zlp) */
	is_short = (count != EP0_FIFO_SIZE);

	DBG(DBG_VERY_NOISY, "ep0in %d bytes %d left %p\n", count,
		req->req.length - req->req.actual, req);

	if (unlikely (is_short)) {
		if (ep->dev->req_pending)
			ep0start(ep->dev, UDCCS0_IPR, "short IN");
		else
			UDCCS0 = UDCCS0_IPR;

		count = req->req.length;
		done (ep, req, 0);
		ep0_idle(ep->dev);
#ifndef CONFIG_ARCH_IXP4XX
#if 1
		/* This seems to get rid of lost status irqs in some cases:
		 * host responds quickly, or next request involves config
		 * change automagic, or should have been hidden, or ...
		 *
		 * FIXME get rid of all udelays possible...
		 */
		if (count >= EP0_FIFO_SIZE) {
			count = 100;
			do {
				if ((UDCCS0 & UDCCS0_OPR) != 0) {
					/* clear OPR, generate ack */
					UDCCS0 = UDCCS0_OPR;
					break;
				}
				count--;
				udelay(1);
			} while (count);
		}
#endif
#endif
	} else if (ep->dev->req_pending)
		ep0start(ep->dev, 0, "IN");
	return is_short;
}


/*
 * read_fifo -  unload packet(s) from the fifo we use for usb OUT
 * transfers and put them into the request.  caller should have made
 * sure there's at least one packet ready.
 *
 * returns true if the request completed because of short packet or the
 * request buffer having filled (and maybe overran till end-of-packet).
 */
static int
read_fifo (struct pxa25x_ep *ep, struct pxa25x_request *req)
{
	for (;;) {
		u32		udccs;
		u8		*buf;
		unsigned	bufferspace, count, is_short;

		/* make sure there's a packet in the FIFO.
		 * UDCCS_{BO,IO}_RPC are all the same bit value.
		 * UDCCS_{BO,IO}_RNE are all the same bit value.
		 */
		udccs = *ep->reg_udccs;
		if (unlikely ((udccs & UDCCS_BO_RPC) == 0))
			break;
		buf = req->req.buf + req->req.actual;
		prefetchw(buf);
		bufferspace = req->req.length - req->req.actual;

		/* read all bytes from this packet */
		if (likely (udccs & UDCCS_BO_RNE)) {
			count = 1 + (0x0ff & *ep->reg_ubcr);
			req->req.actual += min (count, bufferspace);
		} else /* zlp */
			count = 0;
		is_short = (count < ep->ep.maxpacket);
		DBG(DBG_VERY_NOISY, "read %s %02x, %d bytes%s req %p %d/%d\n",
			ep->ep.name, udccs, count,
			is_short ? "/S" : "",
			req, req->req.actual, req->req.length);
		while (likely (count-- != 0)) {
			u8	byte = (u8) *ep->reg_uddr;

			if (unlikely (bufferspace == 0)) {
				/* this happens when the driver's buffer
				 * is smaller than what the host sent.
				 * discard the extra data.
				 */
				if (req->req.status != -EOVERFLOW)
					DMSG("%s overflow %d\n",
						ep->ep.name, count);
				req->req.status = -EOVERFLOW;
			} else {
				*buf++ = byte;
				bufferspace--;
			}
		}
		*ep->reg_udccs =  UDCCS_BO_RPC;
		/* RPC/RSP/RNE could now reflect the other packet buffer */

		/* iso is one request per packet */
		if (ep->bmAttributes == USB_ENDPOINT_XFER_ISOC) {
			if (udccs & UDCCS_IO_ROF)
				req->req.status = -EHOSTUNREACH;
			/* more like "is_done" */
			is_short = 1;
		}

		/* completion */
		if (is_short || req->req.actual == req->req.length) {
			done (ep, req, 0);
			if (list_empty(&ep->queue))
				pio_irq_disable (ep->bEndpointAddress);
			return 1;
		}

		/* finished that packet.  the next one may be waiting... */
	}
	return 0;
}

/*
 * special ep0 version of the above.  no UBCR0 or double buffering; status
 * handshaking is magic.  most device protocols don't need control-OUT.
 * CDC vendor commands (and RNDIS), mass storage CB/CBI, and some other
 * protocols do use them.
 */
static int
read_ep0_fifo (struct pxa25x_ep *ep, struct pxa25x_request *req)
{
	u8		*buf, byte;
	unsigned	bufferspace;

	buf = req->req.buf + req->req.actual;
	bufferspace = req->req.length - req->req.actual;

	while (UDCCS0 & UDCCS0_RNE) {
		byte = (u8) UDDR0;

		if (unlikely (bufferspace == 0)) {
			/* this happens when the driver's buffer
			 * is smaller than what the host sent.
			 * discard the extra data.
			 */
			if (req->req.status != -EOVERFLOW)
				DMSG("%s overflow\n", ep->ep.name);
			req->req.status = -EOVERFLOW;
		} else {
			*buf++ = byte;
			req->req.actual++;
			bufferspace--;
		}
	}

	UDCCS0 = UDCCS0_OPR | UDCCS0_IPR;

	/* completion */
	if (req->req.actual >= req->req.length)
		return 1;

	/* finished that packet.  the next one may be waiting... */
	return 0;
}

/*-------------------------------------------------------------------------*/

static int
pxa25x_ep_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
{
	struct pxa25x_request	*req;
	struct pxa25x_ep	*ep;
	struct pxa25x_udc	*dev;
	unsigned long		flags;

	req = container_of(_req, struct pxa25x_request, req);
	if (unlikely (!_req || !_req->complete || !_req->buf
			|| !list_empty(&req->queue))) {
		DMSG("%s, bad params\n", __func__);
		return -EINVAL;
	}

	ep = container_of(_ep, struct pxa25x_ep, ep);
	if (unlikely (!_ep || (!ep->desc && ep->ep.name != ep0name))) {
		DMSG("%s, bad ep\n", __func__);
		return -EINVAL;
	}

	dev = ep->dev;
	if (unlikely (!dev->driver
			|| dev->gadget.speed == USB_SPEED_UNKNOWN)) {
		DMSG("%s, bogus device state\n", __func__);
		return -ESHUTDOWN;
	}

	/* iso is always one packet per request, that's the only way
	 * we can report per-packet status.  that also helps with dma.
	 */
	if (unlikely (ep->bmAttributes == USB_ENDPOINT_XFER_ISOC
			&& req->req.length > le16_to_cpu
						(ep->desc->wMaxPacketSize)))
		return -EMSGSIZE;

	DBG(DBG_NOISY, "%s queue req %p, len %d buf %p\n",
		_ep->name, _req, _req->length, _req->buf);

	local_irq_save(flags);

	_req->status = -EINPROGRESS;
	_req->actual = 0;

	/* kickstart this i/o queue? */
	if (list_empty(&ep->queue) && !ep->stopped) {
		if (ep->desc == NULL/* ep0 */) {
			unsigned	length = _req->length;

			switch (dev->ep0state) {
			case EP0_IN_DATA_PHASE:
				dev->stats.write.ops++;
				if (write_ep0_fifo(ep, req))
					req = NULL;
				break;

			case EP0_OUT_DATA_PHASE:
				dev->stats.read.ops++;
				/* messy ... */
				if (dev->req_config) {
					DBG(DBG_VERBOSE, "ep0 config ack%s\n",
						dev->has_cfr ?  "" : " raced");
					if (dev->has_cfr)
						UDCCFR = UDCCFR_AREN|UDCCFR_ACM
							|UDCCFR_MB1;
					done(ep, req, 0);
					dev->ep0state = EP0_END_XFER;
					local_irq_restore (flags);
					return 0;
				}
				if (dev->req_pending)
					ep0start(dev, UDCCS0_IPR, "OUT");
				if (length == 0 || ((UDCCS0 & UDCCS0_RNE) != 0
						&& read_ep0_fifo(ep, req))) {
					ep0_idle(dev);
					done(ep, req, 0);
					req = NULL;
				}
				break;

			default:
				DMSG("ep0 i/o, odd state %d\n", dev->ep0state);
				local_irq_restore (flags);
				return -EL2HLT;
			}
		/* can the FIFO can satisfy the request immediately? */
		} else if ((ep->bEndpointAddress & USB_DIR_IN) != 0) {
			if ((*ep->reg_udccs & UDCCS_BI_TFS) != 0
					&& write_fifo(ep, req))
				req = NULL;
		} else if ((*ep->reg_udccs & UDCCS_BO_RFS) != 0
				&& read_fifo(ep, req)) {
			req = NULL;
		}

		if (likely (req && ep->desc))
			pio_irq_enable(ep->bEndpointAddress);
	}

	/* pio or dma irq handler advances the queue. */
	if (likely(req != NULL))
		list_add_tail(&req->queue, &ep->queue);
	local_irq_restore(flags);

	return 0;
}


/*
 *	nuke - dequeue ALL requests
 */
static void nuke(struct pxa25x_ep *ep, int status)
{
	struct pxa25x_request *req;

	/* called with irqs blocked */
	while (!list_empty(&ep->queue)) {
		req = list_entry(ep->queue.next,
				struct pxa25x_request,
				queue);
		done(ep, req, status);
	}
	if (ep->desc)
		pio_irq_disable (ep->bEndpointAddress);
}


/* dequeue JUST ONE request */
static int pxa25x_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
{
	struct pxa25x_ep	*ep;
	struct pxa25x_request	*req;
	unsigned long		flags;

	ep = container_of(_ep, struct pxa25x_ep, ep);
	if (!_ep || ep->ep.name == ep0name)
		return -EINVAL;

	local_irq_save(flags);

	/* make sure it's actually queued on this endpoint */
	list_for_each_entry (req, &ep->queue, queue) {
		if (&req->req == _req)
			break;
	}
	if (&req->req != _req) {
		local_irq_restore(flags);
		return -EINVAL;
	}

	done(ep, req, -ECONNRESET);

	local_irq_restore(flags);
	return 0;
}

/*-------------------------------------------------------------------------*/

static int pxa25x_ep_set_halt(struct usb_ep *_ep, int value)
{
	struct pxa25x_ep	*ep;
	unsigned long		flags;

	ep = container_of(_ep, struct pxa25x_ep, ep);
	if (unlikely (!_ep
			|| (!ep->desc && ep->ep.name != ep0name))
			|| ep->bmAttributes == USB_ENDPOINT_XFER_ISOC) {
		DMSG("%s, bad ep\n", __func__);
		return -EINVAL;
	}
	if (value == 0) {
		/* this path (reset toggle+halt) is needed to implement
		 * SET_INTERFACE on normal hardware.  but it can't be
		 * done from software on the PXA UDC, and the hardware
		 * forgets to do it as part of SET_INTERFACE automagic.
		 */
		DMSG("only host can clear %s halt\n", _ep->name);
		return -EROFS;
	}

	local_irq_save(flags);

	if ((ep->bEndpointAddress & USB_DIR_IN) != 0
			&& ((*ep->reg_udccs & UDCCS_BI_TFS) == 0
			   || !list_empty(&ep->queue))) {
		local_irq_restore(flags);
		return -EAGAIN;
	}

	/* FST bit is the same for control, bulk in, bulk out, interrupt in */
	*ep->reg_udccs = UDCCS_BI_FST|UDCCS_BI_FTF;

	/* ep0 needs special care */
	if (!ep->desc) {
		start_watchdog(ep->dev);
		ep->dev->req_pending = 0;
		ep->dev->ep0state = EP0_STALL;

	/* and bulk/intr endpoints like dropping stalls too */
	} else {
		unsigned i;
		for (i = 0; i < 1000; i += 20) {
			if (*ep->reg_udccs & UDCCS_BI_SST)
				break;
			udelay(20);
		}
	}
	local_irq_restore(flags);

	DBG(DBG_VERBOSE, "%s halt\n", _ep->name);
	return 0;
}

static int pxa25x_ep_fifo_status(struct usb_ep *_ep)
{
	struct pxa25x_ep        *ep;

	ep = container_of(_ep, struct pxa25x_ep, ep);
	if (!_ep) {
		DMSG("%s, bad ep\n", __func__);
		return -ENODEV;
	}
	/* pxa can't report unclaimed bytes from IN fifos */
	if ((ep->bEndpointAddress & USB_DIR_IN) != 0)
		return -EOPNOTSUPP;
	if (ep->dev->gadget.speed == USB_SPEED_UNKNOWN
			|| (*ep->reg_udccs & UDCCS_BO_RFS) == 0)
		return 0;
	else
		return (*ep->reg_ubcr & 0xfff) + 1;
}

static void pxa25x_ep_fifo_flush(struct usb_ep *_ep)
{
	struct pxa25x_ep        *ep;

	ep = container_of(_ep, struct pxa25x_ep, ep);
	if (!_ep || ep->ep.name == ep0name || !list_empty(&ep->queue)) {
		DMSG("%s, bad ep\n", __func__);
		return;
	}

	/* toggle and halt bits stay unchanged */

	/* for OUT, just read and discard the FIFO contents. */
	if ((ep->bEndpointAddress & USB_DIR_IN) == 0) {
		while (((*ep->reg_udccs) & UDCCS_BO_RNE) != 0)
			(void) *ep->reg_uddr;
		return;
	}

	/* most IN status is the same, but ISO can't stall */
	*ep->reg_udccs = UDCCS_BI_TPC|UDCCS_BI_FTF|UDCCS_BI_TUR
		| (ep->bmAttributes == USB_ENDPOINT_XFER_ISOC)
			? 0 : UDCCS_BI_SST;
}


static struct usb_ep_ops pxa25x_ep_ops = {
	.enable		= pxa25x_ep_enable,
	.disable	= pxa25x_ep_disable,

	.alloc_request	= pxa25x_ep_alloc_request,
	.free_request	= pxa25x_ep_free_request,

	.queue		= pxa25x_ep_queue,
	.dequeue	= pxa25x_ep_dequeue,

	.set_halt	= pxa25x_ep_set_halt,
	.fifo_status	= pxa25x_ep_fifo_status,
	.fifo_flush	= pxa25x_ep_fifo_flush,
};


/* ---------------------------------------------------------------------------
 *	device-scoped parts of the api to the usb controller hardware
 * ---------------------------------------------------------------------------
 */

static int pxa25x_udc_get_frame(struct usb_gadget *_gadget)
{
	return ((UFNRH & 0x07) << 8) | (UFNRL & 0xff);
}

static int pxa25x_udc_wakeup(struct usb_gadget *_gadget)
{
	/* host may not have enabled remote wakeup */
	if ((UDCCS0 & UDCCS0_DRWF) == 0)
		return -EHOSTUNREACH;
	udc_set_mask_UDCCR(UDCCR_RSM);
	return 0;
}

static void stop_activity(struct pxa25x_udc *, struct usb_gadget_driver *);
static void udc_enable (struct pxa25x_udc *);
static void udc_disable(struct pxa25x_udc *);

/* We disable the UDC -- and its 48 MHz clock -- whenever it's not
 * in active use.
 */
static int pullup(struct pxa25x_udc *udc)
{
	int is_active = udc->vbus && udc->pullup && !udc->suspended;
	DMSG("%s\n", is_active ? "active" : "inactive");
	if (is_active) {
		if (!udc->active) {
			udc->active = 1;
			/* Enable clock for USB device */
			clk_enable(udc->clk);
			udc_enable(udc);
		}
	} else {
		if (udc->active) {
			if (udc->gadget.speed != USB_SPEED_UNKNOWN) {
				DMSG("disconnect %s\n", udc->driver
					? udc->driver->driver.name
					: "(no driver)");
				stop_activity(udc, udc->driver);
			}
			udc_disable(udc);
			/* Disable clock for USB device */
			clk_disable(udc->clk);
			udc->active = 0;
		}

	}
	return 0;
}

/* VBUS reporting logically comes from a transceiver */
static int pxa25x_udc_vbus_session(struct usb_gadget *_gadget, int is_active)
{
	struct pxa25x_udc	*udc;

	udc = container_of(_gadget, struct pxa25x_udc, gadget);
	udc->vbus = is_active;
	DMSG("vbus %s\n", is_active ? "supplied" : "inactive");
	pullup(udc);
	return 0;
}

/* drivers may have software control over D+ pullup */
static int pxa25x_udc_pullup(struct usb_gadget *_gadget, int is_active)
{
	struct pxa25x_udc	*udc;

	udc = container_of(_gadget, struct pxa25x_udc, gadget);

	/* not all boards support pullup control */
	if (!udc->mach->gpio_pullup && !udc->mach->udc_command)
		return -EOPNOTSUPP;

	udc->pullup = (is_active != 0);
	pullup(udc);
	return 0;
}

static const struct usb_gadget_ops pxa25x_udc_ops = {
	.get_frame	= pxa25x_udc_get_frame,
	.wakeup		= pxa25x_udc_wakeup,
	.vbus_session	= pxa25x_udc_vbus_session,
	.pullup		= pxa25x_udc_pullup,

	// .vbus_draw ... boards may consume current from VBUS, up to
	// 100-500mA based on config.  the 500uA suspend ceiling means
	// that exclusively vbus-powered PXA designs violate USB specs.
};

/*-------------------------------------------------------------------------*/

#ifdef CONFIG_USB_GADGET_DEBUG_FS

static int
udc_seq_show(struct seq_file *m, void *_d)
{
	struct pxa25x_udc	*dev = m->private;
	unsigned long		flags;
	int			i;
	u32			tmp;

	local_irq_save(flags);

	/* basic device status */
	seq_printf(m, DRIVER_DESC "\n"
		"%s version: %s\nGadget driver: %s\nHost %s\n\n",
		driver_name, DRIVER_VERSION SIZE_STR "(pio)",
		dev->driver ? dev->driver->driver.name : "(none)",
		is_vbus_present() ? "full speed" : "disconnected");

	/* registers for device and ep0 */
	seq_printf(m,
		"uicr %02X.%02X, usir %02X.%02x, ufnr %02X.%02X\n",
		UICR1, UICR0, USIR1, USIR0, UFNRH, UFNRL);

	tmp = UDCCR;
	seq_printf(m,
		"udccr %02X =%s%s%s%s%s%s%s%s\n", tmp,
		(tmp & UDCCR_REM) ? " rem" : "",
		(tmp & UDCCR_RSTIR) ? " rstir" : "",
		(tmp & UDCCR_SRM) ? " srm" : "",
		(tmp & UDCCR_SUSIR) ? " susir" : "",
		(tmp & UDCCR_RESIR) ? " resir" : "",
		(tmp & UDCCR_RSM) ? " rsm" : "",
		(tmp & UDCCR_UDA) ? " uda" : "",
		(tmp & UDCCR_UDE) ? " ude" : "");

	tmp = UDCCS0;
	seq_printf(m,
		"udccs0 %02X =%s%s%s%s%s%s%s%s\n", tmp,
		(tmp & UDCCS0_SA) ? " sa" : "",
		(tmp & UDCCS0_RNE) ? " rne" : "",
		(tmp & UDCCS0_FST) ? " fst" : "",
		(tmp & UDCCS0_SST) ? " sst" : "",
		(tmp & UDCCS0_DRWF) ? " dwrf" : "",
		(tmp & UDCCS0_FTF) ? " ftf" : "",
		(tmp & UDCCS0_IPR) ? " ipr" : "",
		(tmp & UDCCS0_OPR) ? " opr" : "");

	if (dev->has_cfr) {
		tmp = UDCCFR;
		seq_printf(m,
			"udccfr %02X =%s%s\n", tmp,
			(tmp & UDCCFR_AREN) ? " aren" : "",
			(tmp & UDCCFR_ACM) ? " acm" : "");
	}

	if (!is_vbus_present() || !dev->driver)
		goto done;

	seq_printf(m, "ep0 IN %lu/%lu, OUT %lu/%lu\nirqs %lu\n\n",
		dev->stats.write.bytes, dev->stats.write.ops,
		dev->stats.read.bytes, dev->stats.read.ops,
		dev->stats.irqs);

	/* dump endpoint queues */
	for (i = 0; i < PXA_UDC_NUM_ENDPOINTS; i++) {
		struct pxa25x_ep	*ep = &dev->ep [i];
		struct pxa25x_request	*req;

		if (i != 0) {
			const struct usb_endpoint_descriptor	*desc;

			desc = ep->desc;
			if (!desc)
				continue;
			tmp = *dev->ep [i].reg_udccs;
			seq_printf(m,
				"%s max %d %s udccs %02x irqs %lu\n",
				ep->ep.name, le16_to_cpu(desc->wMaxPacketSize),
				"pio", tmp, ep->pio_irqs);
			/* TODO translate all five groups of udccs bits! */

		} else /* ep0 should only have one transfer queued */
			seq_printf(m, "ep0 max 16 pio irqs %lu\n",
				ep->pio_irqs);

		if (list_empty(&ep->queue)) {
			seq_printf(m, "\t(nothing queued)\n");
			continue;
		}
		list_for_each_entry(req, &ep->queue, queue) {
			seq_printf(m,
					"\treq %p len %d/%d buf %p\n",
					&req->req, req->req.actual,
					req->req.length, req->req.buf);
		}
	}

done:
	local_irq_restore(flags);
	return 0;
}

static int
udc_debugfs_open(struct inode *inode, struct file *file)
{
	return single_open(file, udc_seq_show, inode->i_private);
}

static const struct file_operations debug_fops = {
	.open		= udc_debugfs_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
	.owner		= THIS_MODULE,
};

#define create_debug_files(dev) \
	do { \
		dev->debugfs_udc = debugfs_create_file(dev->gadget.name, \
			S_IRUGO, NULL, dev, &debug_fops); \
	} while (0)
#define remove_debug_files(dev) \
	do { \
		if (dev->debugfs_udc) \
			debugfs_remove(dev->debugfs_udc); \
	} while (0)

#else	/* !CONFIG_USB_GADGET_DEBUG_FILES */

#define create_debug_files(dev) do {} while (0)
#define remove_debug_files(dev) do {} while (0)

#endif	/* CONFIG_USB_GADGET_DEBUG_FILES */

/*-------------------------------------------------------------------------*/

/*
 *	udc_disable - disable USB device controller
 */
static void udc_disable(struct pxa25x_udc *dev)
{
	/* block all irqs */
	udc_set_mask_UDCCR(UDCCR_SRM|UDCCR_REM);
	UICR0 = UICR1 = 0xff;
	UFNRH = UFNRH_SIM;

	/* if hardware supports it, disconnect from usb */
	pullup_off();

	udc_clear_mask_UDCCR(UDCCR_UDE);

	ep0_idle (dev);
	dev->gadget.speed = USB_SPEED_UNKNOWN;
}


/*
 *	udc_reinit - initialize software state
 */
static void udc_reinit(struct pxa25x_udc *dev)
{
	u32	i;

	/* device/ep0 records init */
	INIT_LIST_HEAD (&dev->gadget.ep_list);
	INIT_LIST_HEAD (&dev->gadget.ep0->ep_list);
	dev->ep0state = EP0_IDLE;

	/* basic endpoint records init */
	for (i = 0; i < PXA_UDC_NUM_ENDPOINTS; i++) {
		struct pxa25x_ep *ep = &dev->ep[i];

		if (i != 0)
			list_add_tail (&ep->ep.ep_list, &dev->gadget.ep_list);

		ep->desc = NULL;
		ep->stopped = 0;
		INIT_LIST_HEAD (&ep->queue);
		ep->pio_irqs = 0;
	}

	/* the rest was statically initialized, and is read-only */
}

/* until it's enabled, this UDC should be completely invisible
 * to any USB host.
 */
static void udc_enable (struct pxa25x_udc *dev)
{
	udc_clear_mask_UDCCR(UDCCR_UDE);

	/* try to clear these bits before we enable the udc */
	udc_ack_int_UDCCR(UDCCR_SUSIR|/*UDCCR_RSTIR|*/UDCCR_RESIR);

	ep0_idle(dev);
	dev->gadget.speed = USB_SPEED_UNKNOWN;
	dev->stats.irqs = 0;

	/*
	 * sequence taken from chapter 12.5.10, PXA250 AppProcDevManual:
	 * - enable UDC
	 * - if RESET is already in progress, ack interrupt
	 * - unmask reset interrupt
	 */
	udc_set_mask_UDCCR(UDCCR_UDE);
	if (!(UDCCR & UDCCR_UDA))
		udc_ack_int_UDCCR(UDCCR_RSTIR);

	if (dev->has_cfr /* UDC_RES2 is defined */) {
		/* pxa255 (a0+) can avoid a set_config race that could
		 * prevent gadget drivers from configuring correctly
		 */
		UDCCFR = UDCCFR_ACM | UDCCFR_MB1;
	} else {
		/* "USB test mode" for pxa250 errata 40-42 (stepping a0, a1)
		 * which could result in missing packets and interrupts.
		 * supposedly one bit per endpoint, controlling whether it
		 * double buffers or not; ACM/AREN bits fit into the holes.
		 * zero bits (like USIR0_IRx) disable double buffering.
		 */
		UDC_RES1 = 0x00;
		UDC_RES2 = 0x00;
	}

	/* enable suspend/resume and reset irqs */
	udc_clear_mask_UDCCR(UDCCR_SRM | UDCCR_REM);

	/* enable ep0 irqs */
	UICR0 &= ~UICR0_IM0;

	/* if hardware supports it, pullup D+ and wait for reset */
	pullup_on();
}


/* when a driver is successfully registered, it will receive
 * control requests including set_configuration(), which enables
 * non-control requests.  then usb traffic follows until a
 * disconnect is reported.  then a host may connect again, or
 * the driver might get unbound.
 */
int usb_gadget_register_driver(struct usb_gadget_driver *driver)
{
	struct pxa25x_udc	*dev = the_controller;
	int			retval;

	if (!driver
			|| driver->speed < USB_SPEED_FULL
			|| !driver->bind
			|| !driver->disconnect
			|| !driver->setup)
		return -EINVAL;
	if (!dev)
		return -ENODEV;
	if (dev->driver)
		return -EBUSY;

	/* first hook up the driver ... */
	dev->driver = driver;
	dev->gadget.dev.driver = &driver->driver;
	dev->pullup = 1;

	retval = device_add (&dev->gadget.dev);
	if (retval) {
fail:
		dev->driver = NULL;
		dev->gadget.dev.driver = NULL;
		return retval;
	}
	retval = driver->bind(&dev->gadget);
	if (retval) {
		DMSG("bind to driver %s --> error %d\n",
				driver->driver.name, retval);
		device_del (&dev->gadget.dev);
		goto fail;
	}

	/* ... then enable host detection and ep0; and we're ready
	 * for set_configuration as well as eventual disconnect.
	 */
	DMSG("registered gadget driver '%s'\n", driver->driver.name);
	pullup(dev);
	dump_state(dev);
	return 0;
}
EXPORT_SYMBOL(usb_gadget_register_driver);

static void
stop_activity(struct pxa25x_udc *dev, struct usb_gadget_driver *driver)
{
	int i;

	/* don't disconnect drivers more than once */
	if (dev->gadget.speed == USB_SPEED_UNKNOWN)
		driver = NULL;
	dev->gadget.speed = USB_SPEED_UNKNOWN;

	/* prevent new request submissions, kill any outstanding requests  */
	for (i = 0; i < PXA_UDC_NUM_ENDPOINTS; i++) {
		struct pxa25x_ep *ep = &dev->ep[i];

		ep->stopped = 1;
		nuke(ep, -ESHUTDOWN);
	}
	del_timer_sync(&dev->timer);

	/* report disconnect; the driver is already quiesced */
	if (driver)
		driver->disconnect(&dev->gadget);

	/* re-init driver-visible data structures */
	udc_reinit(dev);
}

int usb_gadget_unregister_driver(struct usb_gadget_driver *driver)
{
	struct pxa25x_udc	*dev = the_controller;

	if (!dev)
		return -ENODEV;
	if (!driver || driver != dev->driver || !driver->unbind)
		return -EINVAL;

	local_irq_disable();
	dev->pullup = 0;
	pullup(dev);
	stop_activity(dev, driver);
	local_irq_enable();

	driver->unbind(&dev->gadget);
	dev->gadget.dev.driver = NULL;
	dev->driver = NULL;

	device_del (&dev->gadget.dev);

	DMSG("unregistered gadget driver '%s'\n", driver->driver.name);
	dump_state(dev);
	return 0;
}
EXPORT_SYMBOL(usb_gadget_unregister_driver);


/*-------------------------------------------------------------------------*/

#ifdef CONFIG_ARCH_LUBBOCK

/* Lubbock has separate connect and disconnect irqs.  More typical designs
 * use one GPIO as the VBUS IRQ, and another to control the D+ pullup.
 */

static irqreturn_t
lubbock_vbus_irq(int irq, void *_dev)
{
	struct pxa25x_udc	*dev = _dev;
	int			vbus;

	dev->stats.irqs++;
	switch (irq) {
	case LUBBOCK_USB_IRQ:
		vbus = 1;
		disable_irq(LUBBOCK_USB_IRQ);
		enable_irq(LUBBOCK_USB_DISC_IRQ);
		break;
	case LUBBOCK_USB_DISC_IRQ:
		vbus = 0;
		disable_irq(LUBBOCK_USB_DISC_IRQ);
		enable_irq(LUBBOCK_USB_IRQ);
		break;
	default:
		return IRQ_NONE;
	}

	pxa25x_udc_vbus_session(&dev->gadget, vbus);
	return IRQ_HANDLED;
}

#endif

static irqreturn_t udc_vbus_irq(int irq, void *_dev)
{
	struct pxa25x_udc	*dev = _dev;

	pxa25x_udc_vbus_session(&dev->gadget, is_vbus_present());
	return IRQ_HANDLED;
}


/*-------------------------------------------------------------------------*/

static inline void clear_ep_state (struct pxa25x_udc *dev)
{
	unsigned i;

	/* hardware SET_{CONFIGURATION,INTERFACE} automagic resets endpoint
	 * fifos, and pending transactions mustn't be continued in any case.
	 */
	for (i = 1; i < PXA_UDC_NUM_ENDPOINTS; i++)
		nuke(&dev->ep[i], -ECONNABORTED);
}

static void udc_watchdog(unsigned long _dev)
{
	struct pxa25x_udc	*dev = (void *)_dev;

	local_irq_disable();
	if (dev->ep0state == EP0_STALL
			&& (UDCCS0 & UDCCS0_FST) == 0
			&& (UDCCS0 & UDCCS0_SST) == 0) {
		UDCCS0 = UDCCS0_FST|UDCCS0_FTF;
		DBG(DBG_VERBOSE, "ep0 re-stall\n");
		start_watchdog(dev);
	}
	local_irq_enable();
}

static void handle_ep0 (struct pxa25x_udc *dev)
{
	u32			udccs0 = UDCCS0;
	struct pxa25x_ep	*ep = &dev->ep [0];
	struct pxa25x_request	*req;
	union {
		struct usb_ctrlrequest	r;
		u8			raw [8];
		u32			word [2];
	} u;

	if (list_empty(&ep->queue))
		req = NULL;
	else
		req = list_entry(ep->queue.next, struct pxa25x_request, queue);

	/* clear stall status */
	if (udccs0 & UDCCS0_SST) {
		nuke(ep, -EPIPE);
		UDCCS0 = UDCCS0_SST;
		del_timer(&dev->timer);
		ep0_idle(dev);
	}

	/* previous request unfinished?  non-error iff back-to-back ... */
	if ((udccs0 & UDCCS0_SA) != 0 && dev->ep0state != EP0_IDLE) {
		nuke(ep, 0);
		del_timer(&dev->timer);
		ep0_idle(dev);
	}

	switch (dev->ep0state) {
	case EP0_IDLE:
		/* late-breaking status? */
		udccs0 = UDCCS0;

		/* start control request? */
		if (likely((udccs0 & (UDCCS0_OPR|UDCCS0_SA|UDCCS0_RNE))
				== (UDCCS0_OPR|UDCCS0_SA|UDCCS0_RNE))) {
			int i;

			nuke (ep, -EPROTO);

			/* read SETUP packet */
			for (i = 0; i < 8; i++) {
				if (unlikely(!(UDCCS0 & UDCCS0_RNE))) {
bad_setup:
					DMSG("SETUP %d!\n", i);
					goto stall;
				}
				u.raw [i] = (u8) UDDR0;
			}
			if (unlikely((UDCCS0 & UDCCS0_RNE) != 0))
				goto bad_setup;

got_setup:
			DBG(DBG_VERBOSE, "SETUP %02x.%02x v%04x i%04x l%04x\n",
				u.r.bRequestType, u.r.bRequest,
				le16_to_cpu(u.r.wValue),
				le16_to_cpu(u.r.wIndex),
				le16_to_cpu(u.r.wLength));

			/* cope with automagic for some standard requests. */
			dev->req_std = (u.r.bRequestType & USB_TYPE_MASK)
						== USB_TYPE_STANDARD;
			dev->req_config = 0;
			dev->req_pending = 1;
			switch (u.r.bRequest) {
			/* hardware restricts gadget drivers here! */
			case USB_REQ_SET_CONFIGURATION:
				if (u.r.bRequestType == USB_RECIP_DEVICE) {
					/* reflect hardware's automagic
					 * up to the gadget driver.
					 */
config_change:
					dev->req_config = 1;
					clear_ep_state(dev);
					/* if !has_cfr, there's no synch
					 * else use AREN (later) not SA|OPR
					 * USIR0_IR0 acts edge sensitive
					 */
				}
				break;
			/* ... and here, even more ... */
			case USB_REQ_SET_INTERFACE:
				if (u.r.bRequestType == USB_RECIP_INTERFACE) {
					/* udc hardware is broken by design:
					 *  - altsetting may only be zero;
					 *  - hw resets all interfaces' eps;
					 *  - ep reset doesn't include halt(?).
					 */
					DMSG("broken set_interface (%d/%d)\n",
						le16_to_cpu(u.r.wIndex),
						le16_to_cpu(u.r.wValue));
					goto config_change;
				}
				break;
			/* hardware was supposed to hide this */
			case USB_REQ_SET_ADDRESS:
				if (u.r.bRequestType == USB_RECIP_DEVICE) {
					ep0start(dev, 0, "address");
					return;
				}
				break;
			}

			if (u.r.bRequestType & USB_DIR_IN)
				dev->ep0state = EP0_IN_DATA_PHASE;
			else
				dev->ep0state = EP0_OUT_DATA_PHASE;

			i = dev->driver->setup(&dev->gadget, &u.r);
			if (i < 0) {
				/* hardware automagic preventing STALL... */
				if (dev->req_config) {
					/* hardware sometimes neglects to tell
					 * tell us about config change events,
					 * so later ones may fail...
					 */
					WARNING("config change %02x fail %d?\n",
						u.r.bRequest, i);
					return;
					/* TODO experiment:  if has_cfr,
					 * hardware didn't ACK; maybe we
					 * could actually STALL!
					 */
				}
				DBG(DBG_VERBOSE, "protocol STALL, "
					"%02x err %d\n", UDCCS0, i);
stall:
				/* the watchdog timer helps deal with cases
				 * where udc seems to clear FST wrongly, and
				 * then NAKs instead of STALLing.
				 */
				ep0start(dev, UDCCS0_FST|UDCCS0_FTF, "stall");
				start_watchdog(dev);
				dev->ep0state = EP0_STALL;

			/* deferred i/o == no response yet */
			} else if (dev->req_pending) {
				if (likely(dev->ep0state == EP0_IN_DATA_PHASE
						|| dev->req_std || u.r.wLength))
					ep0start(dev, 0, "defer");
				else
					ep0start(dev, UDCCS0_IPR, "defer/IPR");
			}

			/* expect at least one data or status stage irq */
			return;

		} else if (likely((udccs0 & (UDCCS0_OPR|UDCCS0_SA))
				== (UDCCS0_OPR|UDCCS0_SA))) {
			unsigned i;

			/* pxa210/250 erratum 131 for B0/B1 says RNE lies.
			 * still observed on a pxa255 a0.
			 */
			DBG(DBG_VERBOSE, "e131\n");
			nuke(ep, -EPROTO);

			/* read SETUP data, but don't trust it too much */
			for (i = 0; i < 8; i++)
				u.raw [i] = (u8) UDDR0;
			if ((u.r.bRequestType & USB_RECIP_MASK)
					> USB_RECIP_OTHER)
				goto stall;
			if (u.word [0] == 0 && u.word [1] == 0)
				goto stall;
			goto got_setup;
		} else {
			/* some random early IRQ:
			 * - we acked FST
			 * - IPR cleared
			 * - OPR got set, without SA (likely status stage)
			 */
			UDCCS0 = udccs0 & (UDCCS0_SA|UDCCS0_OPR);
		}
		break;
	case EP0_IN_DATA_PHASE:			/* GET_DESCRIPTOR etc */
		if (udccs0 & UDCCS0_OPR) {
			UDCCS0 = UDCCS0_OPR|UDCCS0_FTF;
			DBG(DBG_VERBOSE, "ep0in premature status\n");
			if (req)
				done(ep, req, 0);
			ep0_idle(dev);
		} else /* irq was IPR clearing */ {
			if (req) {
				/* this IN packet might finish the request */
				(void) write_ep0_fifo(ep, req);
			} /* else IN token before response was written */
		}
		break;
	case EP0_OUT_DATA_PHASE:		/* SET_DESCRIPTOR etc */
		if (udccs0 & UDCCS0_OPR) {
			if (req) {
				/* this OUT packet might finish the request */
				if (read_ep0_fifo(ep, req))
					done(ep, req, 0);
				/* else more OUT packets expected */
			} /* else OUT token before read was issued */
		} else /* irq was IPR clearing */ {
			DBG(DBG_VERBOSE, "ep0out premature status\n");
			if (req)
				done(ep, req, 0);
			ep0_idle(dev);
		}
		break;
	case EP0_END_XFER:
		if (req)
			done(ep, req, 0);
		/* ack control-IN status (maybe in-zlp was skipped)
		 * also appears after some config change events.
		 */
		if (udccs0 & UDCCS0_OPR)
			UDCCS0 = UDCCS0_OPR;
		ep0_idle(dev);
		break;
	case EP0_STALL:
		UDCCS0 = UDCCS0_FST;
		break;
	}
	USIR0 = USIR0_IR0;
}

static void handle_ep(struct pxa25x_ep *ep)
{
	struct pxa25x_request	*req;
	int			is_in = ep->bEndpointAddress & USB_DIR_IN;
	int			completed;
	u32			udccs, tmp;

	do {
		completed = 0;
		if (likely (!list_empty(&ep->queue)))
			req = list_entry(ep->queue.next,
					struct pxa25x_request, queue);
		else
			req = NULL;

		// TODO check FST handling

		udccs = *ep->reg_udccs;
		if (unlikely(is_in)) {	/* irq from TPC, SST, or (ISO) TUR */
			tmp = UDCCS_BI_TUR;
			if (likely(ep->bmAttributes == USB_ENDPOINT_XFER_BULK))
				tmp |= UDCCS_BI_SST;
			tmp &= udccs;
			if (likely (tmp))
				*ep->reg_udccs = tmp;
			if (req && likely ((udccs & UDCCS_BI_TFS) != 0))
				completed = write_fifo(ep, req);

		} else {	/* irq from RPC (or for ISO, ROF) */
			if (likely(ep->bmAttributes == USB_ENDPOINT_XFER_BULK))
				tmp = UDCCS_BO_SST | UDCCS_BO_DME;
			else
				tmp = UDCCS_IO_ROF | UDCCS_IO_DME;
			tmp &= udccs;
			if (likely(tmp))
				*ep->reg_udccs = tmp;

			/* fifos can hold packets, ready for reading... */
			if (likely(req)) {
				completed = read_fifo(ep, req);
			} else
				pio_irq_disable (ep->bEndpointAddress);
		}
		ep->pio_irqs++;
	} while (completed);
}

/*
 *	pxa25x_udc_irq - interrupt handler
 *
 * avoid delays in ep0 processing. the control handshaking isn't always
 * under software control (pxa250c0 and the pxa255 are better), and delays
 * could cause usb protocol errors.
 */
static irqreturn_t
pxa25x_udc_irq(int irq, void *_dev)
{
	struct pxa25x_udc	*dev = _dev;
	int			handled;

	dev->stats.irqs++;
	do {
		u32		udccr = UDCCR;

		handled = 0;

		/* SUSpend Interrupt Request */
		if (unlikely(udccr & UDCCR_SUSIR)) {
			udc_ack_int_UDCCR(UDCCR_SUSIR);
			handled = 1;
			DBG(DBG_VERBOSE, "USB suspend%s\n", is_vbus_present()
				? "" : "+disconnect");

			if (!is_vbus_present())
				stop_activity(dev, dev->driver);
			else if (dev->gadget.speed != USB_SPEED_UNKNOWN
					&& dev->driver
					&& dev->driver->suspend)
				dev->driver->suspend(&dev->gadget);
			ep0_idle (dev);
		}

		/* RESume Interrupt Request */
		if (unlikely(udccr & UDCCR_RESIR)) {
			udc_ack_int_UDCCR(UDCCR_RESIR);
			handled = 1;
			DBG(DBG_VERBOSE, "USB resume\n");

			if (dev->gadget.speed != USB_SPEED_UNKNOWN
					&& dev->driver
					&& dev->driver->resume
					&& is_vbus_present())
				dev->driver->resume(&dev->gadget);
		}

		/* ReSeT Interrupt Request - USB reset */
		if (unlikely(udccr & UDCCR_RSTIR)) {
			udc_ack_int_UDCCR(UDCCR_RSTIR);
			handled = 1;

			if ((UDCCR & UDCCR_UDA) == 0) {
				DBG(DBG_VERBOSE, "USB reset start\n");

				/* reset driver and endpoints,
				 * in case that's not yet done
				 */
				stop_activity (dev, dev->driver);

			} else {
				DBG(DBG_VERBOSE, "USB reset end\n");
				dev->gadget.speed = USB_SPEED_FULL;
				memset(&dev->stats, 0, sizeof dev->stats);
				/* driver and endpoints are still reset */
			}

		} else {
			u32	usir0 = USIR0 & ~UICR0;
			u32	usir1 = USIR1 & ~UICR1;
			int	i;

			if (unlikely (!usir0 && !usir1))
				continue;

			DBG(DBG_VERY_NOISY, "irq %02x.%02x\n", usir1, usir0);

			/* control traffic */
			if (usir0 & USIR0_IR0) {
				dev->ep[0].pio_irqs++;
				handle_ep0(dev);
				handled = 1;
			}

			/* endpoint data transfers */
			for (i = 0; i < 8; i++) {
				u32	tmp = 1 << i;

				if (i && (usir0 & tmp)) {
					handle_ep(&dev->ep[i]);
					USIR0 |= tmp;
					handled = 1;
				}
				if (usir1 & tmp) {
					handle_ep(&dev->ep[i+8]);
					USIR1 |= tmp;
					handled = 1;
				}
			}
		}

		/* we could also ask for 1 msec SOF (SIR) interrupts */

	} while (handled);
	return IRQ_HANDLED;
}

/*-------------------------------------------------------------------------*/

static void nop_release (struct device *dev)
{
	DMSG("%s %s\n", __func__, dev_name(dev));
}

/* this uses load-time allocation and initialization (instead of
 * doing it at run-time) to save code, eliminate fault paths, and
 * be more obviously correct.
 */
static struct pxa25x_udc memory = {
	.gadget = {
		.ops		= &pxa25x_udc_ops,
		.ep0		= &memory.ep[0].ep,
		.name		= driver_name,
		.dev = {
			.bus_id		= "gadget",
			.release	= nop_release,
		},
	},

	/* control endpoint */
	.ep[0] = {
		.ep = {
			.name		= ep0name,
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= EP0_FIFO_SIZE,
		},
		.dev		= &memory,
		.reg_udccs	= &UDCCS0,
		.reg_uddr	= &UDDR0,
	},

	/* first group of endpoints */
	.ep[1] = {
		.ep = {
			.name		= "ep1in-bulk",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= BULK_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= BULK_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 1,
		.bmAttributes	= USB_ENDPOINT_XFER_BULK,
		.reg_udccs	= &UDCCS1,
		.reg_uddr	= &UDDR1,
	},
	.ep[2] = {
		.ep = {
			.name		= "ep2out-bulk",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= BULK_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= BULK_FIFO_SIZE,
		.bEndpointAddress = 2,
		.bmAttributes	= USB_ENDPOINT_XFER_BULK,
		.reg_udccs	= &UDCCS2,
		.reg_ubcr	= &UBCR2,
		.reg_uddr	= &UDDR2,
	},
#ifndef CONFIG_USB_PXA25X_SMALL
	.ep[3] = {
		.ep = {
			.name		= "ep3in-iso",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= ISO_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= ISO_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 3,
		.bmAttributes	= USB_ENDPOINT_XFER_ISOC,
		.reg_udccs	= &UDCCS3,
		.reg_uddr	= &UDDR3,
	},
	.ep[4] = {
		.ep = {
			.name		= "ep4out-iso",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= ISO_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= ISO_FIFO_SIZE,
		.bEndpointAddress = 4,
		.bmAttributes	= USB_ENDPOINT_XFER_ISOC,
		.reg_udccs	= &UDCCS4,
		.reg_ubcr	= &UBCR4,
		.reg_uddr	= &UDDR4,
	},
	.ep[5] = {
		.ep = {
			.name		= "ep5in-int",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= INT_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= INT_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 5,
		.bmAttributes	= USB_ENDPOINT_XFER_INT,
		.reg_udccs	= &UDCCS5,
		.reg_uddr	= &UDDR5,
	},

	/* second group of endpoints */
	.ep[6] = {
		.ep = {
			.name		= "ep6in-bulk",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= BULK_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= BULK_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 6,
		.bmAttributes	= USB_ENDPOINT_XFER_BULK,
		.reg_udccs	= &UDCCS6,
		.reg_uddr	= &UDDR6,
	},
	.ep[7] = {
		.ep = {
			.name		= "ep7out-bulk",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= BULK_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= BULK_FIFO_SIZE,
		.bEndpointAddress = 7,
		.bmAttributes	= USB_ENDPOINT_XFER_BULK,
		.reg_udccs	= &UDCCS7,
		.reg_ubcr	= &UBCR7,
		.reg_uddr	= &UDDR7,
	},
	.ep[8] = {
		.ep = {
			.name		= "ep8in-iso",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= ISO_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= ISO_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 8,
		.bmAttributes	= USB_ENDPOINT_XFER_ISOC,
		.reg_udccs	= &UDCCS8,
		.reg_uddr	= &UDDR8,
	},
	.ep[9] = {
		.ep = {
			.name		= "ep9out-iso",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= ISO_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= ISO_FIFO_SIZE,
		.bEndpointAddress = 9,
		.bmAttributes	= USB_ENDPOINT_XFER_ISOC,
		.reg_udccs	= &UDCCS9,
		.reg_ubcr	= &UBCR9,
		.reg_uddr	= &UDDR9,
	},
	.ep[10] = {
		.ep = {
			.name		= "ep10in-int",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= INT_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= INT_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 10,
		.bmAttributes	= USB_ENDPOINT_XFER_INT,
		.reg_udccs	= &UDCCS10,
		.reg_uddr	= &UDDR10,
	},

	/* third group of endpoints */
	.ep[11] = {
		.ep = {
			.name		= "ep11in-bulk",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= BULK_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= BULK_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 11,
		.bmAttributes	= USB_ENDPOINT_XFER_BULK,
		.reg_udccs	= &UDCCS11,
		.reg_uddr	= &UDDR11,
	},
	.ep[12] = {
		.ep = {
			.name		= "ep12out-bulk",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= BULK_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= BULK_FIFO_SIZE,
		.bEndpointAddress = 12,
		.bmAttributes	= USB_ENDPOINT_XFER_BULK,
		.reg_udccs	= &UDCCS12,
		.reg_ubcr	= &UBCR12,
		.reg_uddr	= &UDDR12,
	},
	.ep[13] = {
		.ep = {
			.name		= "ep13in-iso",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= ISO_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= ISO_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 13,
		.bmAttributes	= USB_ENDPOINT_XFER_ISOC,
		.reg_udccs	= &UDCCS13,
		.reg_uddr	= &UDDR13,
	},
	.ep[14] = {
		.ep = {
			.name		= "ep14out-iso",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= ISO_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= ISO_FIFO_SIZE,
		.bEndpointAddress = 14,
		.bmAttributes	= USB_ENDPOINT_XFER_ISOC,
		.reg_udccs	= &UDCCS14,
		.reg_ubcr	= &UBCR14,
		.reg_uddr	= &UDDR14,
	},
	.ep[15] = {
		.ep = {
			.name		= "ep15in-int",
			.ops		= &pxa25x_ep_ops,
			.maxpacket	= INT_FIFO_SIZE,
		},
		.dev		= &memory,
		.fifo_size	= INT_FIFO_SIZE,
		.bEndpointAddress = USB_DIR_IN | 15,
		.bmAttributes	= USB_ENDPOINT_XFER_INT,
		.reg_udccs	= &UDCCS15,
		.reg_uddr	= &UDDR15,
	},
#endif /* !CONFIG_USB_PXA25X_SMALL */
};

#define CP15R0_VENDOR_MASK	0xffffe000

#if	defined(CONFIG_ARCH_PXA)
#define CP15R0_XSCALE_VALUE	0x69052000	/* intel/arm/xscale */

#elif	defined(CONFIG_ARCH_IXP4XX)
#define CP15R0_XSCALE_VALUE	0x69054000	/* intel/arm/ixp4xx */

#endif

#define CP15R0_PROD_MASK	0x000003f0
#define PXA25x			0x00000100	/* and PXA26x */
#define PXA210			0x00000120

#define CP15R0_REV_MASK		0x0000000f

#define CP15R0_PRODREV_MASK	(CP15R0_PROD_MASK | CP15R0_REV_MASK)

#define PXA255_A0		0x00000106	/* or PXA260_B1 */
#define PXA250_C0		0x00000105	/* or PXA26x_B0 */
#define PXA250_B2		0x00000104
#define PXA250_B1		0x00000103	/* or PXA260_A0 */
#define PXA250_B0		0x00000102
#define PXA250_A1		0x00000101
#define PXA250_A0		0x00000100

#define PXA210_C0		0x00000125
#define PXA210_B2		0x00000124
#define PXA210_B1		0x00000123
#define PXA210_B0		0x00000122
#define IXP425_A0		0x000001c1
#define IXP425_B0		0x000001f1
#define IXP465_AD		0x00000200

/*
 *	probe - binds to the platform device
 */
static int __init pxa25x_udc_probe(struct platform_device *pdev)
{
	struct pxa25x_udc *dev = &memory;
	int retval, vbus_irq, irq;
	u32 chiprev;

	/* insist on Intel/ARM/XScale */
	asm("mrc%? p15, 0, %0, c0, c0" : "=r" (chiprev));
	if ((chiprev & CP15R0_VENDOR_MASK) != CP15R0_XSCALE_VALUE) {
		pr_err("%s: not XScale!\n", driver_name);
		return -ENODEV;
	}

	/* trigger chiprev-specific logic */
	switch (chiprev & CP15R0_PRODREV_MASK) {
#if	defined(CONFIG_ARCH_PXA)
	case PXA255_A0:
		dev->has_cfr = 1;
		break;
	case PXA250_A0:
	case PXA250_A1:
		/* A0/A1 "not released"; ep 13, 15 unusable */
		/* fall through */
	case PXA250_B2: case PXA210_B2:
	case PXA250_B1: case PXA210_B1:
	case PXA250_B0: case PXA210_B0:
		/* OUT-DMA is broken ... */
		/* fall through */
	case PXA250_C0: case PXA210_C0:
		break;
#elif	defined(CONFIG_ARCH_IXP4XX)
	case IXP425_A0:
	case IXP425_B0:
	case IXP465_AD:
		dev->has_cfr = 1;
		break;
#endif
	default:
		pr_err("%s: unrecognized processor: %08x\n",
			driver_name, chiprev);
		/* iop3xx, ixp4xx, ... */
		return -ENODEV;
	}

	irq = platform_get_irq(pdev, 0);
	if (irq < 0)
		return -ENODEV;