summaryrefslogtreecommitdiffstats
path: root/drivers/media/pci/ttpci/av7110_av.c
blob: ea9f7d0058a215cf52cac38d64ca9259ce64f475 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * av7110_av.c: audio and video MPEG decoder stuff
 *
 * Copyright (C) 1999-2002 Ralph  Metzler
 *                       & Marcus Metzler for convergence integrated media GmbH
 *
 * originally based on code by:
 * Copyright (C) 1998,1999 Christian Theiss <mistert@rz.fh-augsburg.de>
 *
 * the project's page is at https://linuxtv.org
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/delay.h>
#include <linux/fs.h>

#include "av7110.h"
#include "av7110_hw.h"
#include "av7110_av.h"
#include "av7110_ipack.h"

/* MPEG-2 (ISO 13818 / H.222.0) stream types */
#define PROG_STREAM_MAP  0xBC
#define PRIVATE_STREAM1  0xBD
#define PADDING_STREAM	 0xBE
#define PRIVATE_STREAM2  0xBF
#define AUDIO_STREAM_S	 0xC0
#define AUDIO_STREAM_E	 0xDF
#define VIDEO_STREAM_S	 0xE0
#define VIDEO_STREAM_E	 0xEF
#define ECM_STREAM	 0xF0
#define EMM_STREAM	 0xF1
#define DSM_CC_STREAM	 0xF2
#define ISO13522_STREAM  0xF3
#define PROG_STREAM_DIR  0xFF

#define PTS_DTS_FLAGS	 0xC0

//pts_dts flags
#define PTS_ONLY	 0x80
#define PTS_DTS		 0xC0
#define TS_SIZE		 188
#define TRANS_ERROR	 0x80
#define PAY_START	 0x40
#define TRANS_PRIO	 0x20
#define PID_MASK_HI	 0x1F
//flags
#define TRANS_SCRMBL1	 0x80
#define TRANS_SCRMBL2	 0x40
#define ADAPT_FIELD	 0x20
#define PAYLOAD		 0x10
#define COUNT_MASK	 0x0F

// adaptation flags
#define DISCON_IND	 0x80
#define RAND_ACC_IND	 0x40
#define ES_PRI_IND	 0x20
#define PCR_FLAG	 0x10
#define OPCR_FLAG	 0x08
#define SPLICE_FLAG	 0x04
#define TRANS_PRIV	 0x02
#define ADAP_EXT_FLAG	 0x01

// adaptation extension flags
#define LTW_FLAG	 0x80
#define PIECE_RATE	 0x40
#define SEAM_SPLICE	 0x20


static void p_to_t(u8 const *buf, long int length, u16 pid,
		   u8 *counter, struct dvb_demux_feed *feed);
static int write_ts_to_decoder(struct av7110 *av7110, int type, const u8 *buf, size_t len);


int av7110_record_cb(struct dvb_filter_pes2ts *p2t, u8 *buf, size_t len)
{
	struct dvb_demux_feed *dvbdmxfeed = (struct dvb_demux_feed *) p2t->priv;

	if (!(dvbdmxfeed->ts_type & TS_PACKET))
		return 0;
	if (buf[3] == 0xe0)	 // video PES do not have a length in TS
		buf[4] = buf[5] = 0;
	if (dvbdmxfeed->ts_type & TS_PAYLOAD_ONLY)
		return dvbdmxfeed->cb.ts(buf, len, NULL, 0,
					 &dvbdmxfeed->feed.ts, NULL);
	else
		return dvb_filter_pes2ts(p2t, buf, len, 1);
}

static int dvb_filter_pes2ts_cb(void *priv, unsigned char *data)
{
	struct dvb_demux_feed *dvbdmxfeed = (struct dvb_demux_feed *) priv;

	dvbdmxfeed->cb.ts(data, 188, NULL, 0,
			  &dvbdmxfeed->feed.ts, NULL);
	return 0;
}

int av7110_av_start_record(struct av7110 *av7110, int av,
			   struct dvb_demux_feed *dvbdmxfeed)
{
	int ret = 0;
	struct dvb_demux *dvbdmx = dvbdmxfeed->demux;

	dprintk(2, "av7110:%p, , dvb_demux_feed:%p\n", av7110, dvbdmxfeed);

	if (av7110->playing || (av7110->rec_mode & av))
		return -EBUSY;
	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);
	dvbdmx->recording = 1;
	av7110->rec_mode |= av;

	switch (av7110->rec_mode) {
	case RP_AUDIO:
		dvb_filter_pes2ts_init(&av7110->p2t[0],
				       dvbdmx->pesfilter[0]->pid,
				       dvb_filter_pes2ts_cb,
				       (void *) dvbdmx->pesfilter[0]);
		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, AudioPES, 0);
		break;

	case RP_VIDEO:
		dvb_filter_pes2ts_init(&av7110->p2t[1],
				       dvbdmx->pesfilter[1]->pid,
				       dvb_filter_pes2ts_cb,
				       (void *) dvbdmx->pesfilter[1]);
		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, VideoPES, 0);
		break;

	case RP_AV:
		dvb_filter_pes2ts_init(&av7110->p2t[0],
				       dvbdmx->pesfilter[0]->pid,
				       dvb_filter_pes2ts_cb,
				       (void *) dvbdmx->pesfilter[0]);
		dvb_filter_pes2ts_init(&av7110->p2t[1],
				       dvbdmx->pesfilter[1]->pid,
				       dvb_filter_pes2ts_cb,
				       (void *) dvbdmx->pesfilter[1]);
		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, AV_PES, 0);
		break;
	}
	return ret;
}

int av7110_av_start_play(struct av7110 *av7110, int av)
{
	int ret = 0;
	dprintk(2, "av7110:%p, \n", av7110);

	if (av7110->rec_mode)
		return -EBUSY;
	if (av7110->playing & av)
		return -EBUSY;

	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);

	if (av7110->playing == RP_NONE) {
		av7110_ipack_reset(&av7110->ipack[0]);
		av7110_ipack_reset(&av7110->ipack[1]);
	}

	av7110->playing |= av;
	switch (av7110->playing) {
	case RP_AUDIO:
		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, AudioPES, 0);
		break;
	case RP_VIDEO:
		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, VideoPES, 0);
		av7110->sinfo = 0;
		break;
	case RP_AV:
		av7110->sinfo = 0;
		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, AV_PES, 0);
		break;
	}
	return ret;
}

int av7110_av_stop(struct av7110 *av7110, int av)
{
	int ret = 0;
	dprintk(2, "av7110:%p, \n", av7110);

	if (!(av7110->playing & av) && !(av7110->rec_mode & av))
		return 0;
	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);
	if (av7110->playing) {
		av7110->playing &= ~av;
		switch (av7110->playing) {
		case RP_AUDIO:
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, AudioPES, 0);
			break;
		case RP_VIDEO:
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, VideoPES, 0);
			break;
		case RP_NONE:
			ret = av7110_set_vidmode(av7110, av7110->vidmode);
			break;
		}
	} else {
		av7110->rec_mode &= ~av;
		switch (av7110->rec_mode) {
		case RP_AUDIO:
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, AudioPES, 0);
			break;
		case RP_VIDEO:
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, VideoPES, 0);
			break;
		case RP_NONE:
			break;
		}
	}
	return ret;
}


int av7110_pes_play(void *dest, struct dvb_ringbuffer *buf, int dlen)
{
	int len;
	u32 sync;
	u16 blen;

	if (!dlen) {
		wake_up(&buf->queue);
		return -1;
	}
	while (1) {
		len = dvb_ringbuffer_avail(buf);
		if (len < 6) {
			wake_up(&buf->queue);
			return -1;
		}
		sync =  DVB_RINGBUFFER_PEEK(buf, 0) << 24;
		sync |= DVB_RINGBUFFER_PEEK(buf, 1) << 16;
		sync |= DVB_RINGBUFFER_PEEK(buf, 2) << 8;
		sync |= DVB_RINGBUFFER_PEEK(buf, 3);

		if (((sync &~ 0x0f) == 0x000001e0) ||
		    ((sync &~ 0x1f) == 0x000001c0) ||
		    (sync == 0x000001bd))
			break;
		printk("resync\n");
		DVB_RINGBUFFER_SKIP(buf, 1);
	}
	blen =  DVB_RINGBUFFER_PEEK(buf, 4) << 8;
	blen |= DVB_RINGBUFFER_PEEK(buf, 5);
	blen += 6;
	if (len < blen || blen > dlen) {
		//printk("buffer empty - avail %d blen %u dlen %d\n", len, blen, dlen);
		wake_up(&buf->queue);
		return -1;
	}

	dvb_ringbuffer_read(buf, dest, (size_t) blen);

	dprintk(2, "pread=0x%08lx, pwrite=0x%08lx\n",
	       (unsigned long) buf->pread, (unsigned long) buf->pwrite);
	wake_up(&buf->queue);
	return blen;
}


int av7110_set_volume(struct av7110 *av7110, unsigned int volleft,
		      unsigned int volright)
{
	unsigned int vol, val, balance = 0;
	int err;

	dprintk(2, "av7110:%p, \n", av7110);

	av7110->mixer.volume_left = volleft;
	av7110->mixer.volume_right = volright;

	switch (av7110->adac_type) {
	case DVB_ADAC_TI:
		volleft = (volleft * 256) / 1036;
		volright = (volright * 256) / 1036;
		if (volleft > 0x3f)
			volleft = 0x3f;
		if (volright > 0x3f)
			volright = 0x3f;
		if ((err = SendDAC(av7110, 3, 0x80 + volleft)))
			return err;
		return SendDAC(av7110, 4, volright);

	case DVB_ADAC_CRYSTAL:
		volleft = 127 - volleft / 2;
		volright = 127 - volright / 2;
		i2c_writereg(av7110, 0x20, 0x03, volleft);
		i2c_writereg(av7110, 0x20, 0x04, volright);
		return 0;

	case DVB_ADAC_MSP34x0:
		vol  = (volleft > volright) ? volleft : volright;
		val	= (vol * 0x73 / 255) << 8;
		if (vol > 0)
		       balance = ((volright - volleft) * 127) / vol;
		msp_writereg(av7110, MSP_WR_DSP, 0x0001, balance << 8);
		msp_writereg(av7110, MSP_WR_DSP, 0x0000, val); /* loudspeaker */
		msp_writereg(av7110, MSP_WR_DSP, 0x0006, val); /* headphonesr */
		return 0;

	case DVB_ADAC_MSP34x5:
		vol = (volleft > volright) ? volleft : volright;
		val = (vol * 0x73 / 255) << 8;
		if (vol > 0)
			balance = ((volright - volleft) * 127) / vol;
		msp_writereg(av7110, MSP_WR_DSP, 0x0001, balance << 8);
		msp_writereg(av7110, MSP_WR_DSP, 0x0000, val); /* loudspeaker */
		return 0;
	}

	return 0;
}

int av7110_set_vidmode(struct av7110 *av7110, enum av7110_video_mode mode)
{
	int ret;
	dprintk(2, "av7110:%p, \n", av7110);

	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, LoadVidCode, 1, mode);

	if (!ret && !av7110->playing) {
		ret = ChangePIDs(av7110, av7110->pids[DMX_PES_VIDEO],
			   av7110->pids[DMX_PES_AUDIO],
			   av7110->pids[DMX_PES_TELETEXT],
			   0, av7110->pids[DMX_PES_PCR]);
		if (!ret)
			ret = av7110_fw_cmd(av7110, COMTYPE_PIDFILTER, Scan, 0);
	}
	return ret;
}


static enum av7110_video_mode sw2mode[16] = {
	AV7110_VIDEO_MODE_PAL, AV7110_VIDEO_MODE_NTSC,
	AV7110_VIDEO_MODE_NTSC, AV7110_VIDEO_MODE_PAL,
	AV7110_VIDEO_MODE_NTSC, AV7110_VIDEO_MODE_NTSC,
	AV7110_VIDEO_MODE_PAL, AV7110_VIDEO_MODE_NTSC,
	AV7110_VIDEO_MODE_PAL, AV7110_VIDEO_MODE_PAL,
	AV7110_VIDEO_MODE_PAL, AV7110_VIDEO_MODE_PAL,
	AV7110_VIDEO_MODE_PAL, AV7110_VIDEO_MODE_PAL,
	AV7110_VIDEO_MODE_PAL, AV7110_VIDEO_MODE_PAL,
};

static int get_video_format(struct av7110 *av7110, u8 *buf, int count)
{
	int i;
	int hsize, vsize;
	int sw;
	u8 *p;
	int ret = 0;

	dprintk(2, "av7110:%p, \n", av7110);

	if (av7110->sinfo)
		return 0;
	for (i = 7; i < count - 10; i++) {
		p = buf + i;
		if (p[0] || p[1] || p[2] != 0x01 || p[3] != 0xb3)
			continue;
		p += 4;
		hsize = ((p[1] &0xF0) >> 4) | (p[0] << 4);
		vsize = ((p[1] &0x0F) << 8) | (p[2]);
		sw = (p[3] & 0x0F);
		ret = av7110_set_vidmode(av7110, sw2mode[sw]);
		if (!ret) {
			dprintk(2, "playback %dx%d fr=%d\n", hsize, vsize, sw);
			av7110->sinfo = 1;
		}
		break;
	}
	return ret;
}


/****************************************************************************
 * I/O buffer management and control
 ****************************************************************************/

static inline long aux_ring_buffer_write(struct dvb_ringbuffer *rbuf,
					 const u8 *buf, unsigned long count)
{
	unsigned long todo = count;
	int free;

	while (todo > 0) {
		if (dvb_ringbuffer_free(rbuf) < 2048) {
			if (wait_event_interruptible(rbuf->queue,
						     (dvb_ringbuffer_free(rbuf) >= 2048)))
				return count - todo;
		}
		free = dvb_ringbuffer_free(rbuf);
		if (free > todo)
			free = todo;
		dvb_ringbuffer_write(rbuf, buf, free);
		todo -= free;
		buf += free;
	}

	return count - todo;
}

static void play_video_cb(u8 *buf, int count, void *priv)
{
	struct av7110 *av7110 = (struct av7110 *) priv;
	dprintk(2, "av7110:%p, \n", av7110);

	if ((buf[3] & 0xe0) == 0xe0) {
		get_video_format(av7110, buf, count);
		aux_ring_buffer_write(&av7110->avout, buf, count);
	} else
		aux_ring_buffer_write(&av7110->aout, buf, count);
}

static void play_audio_cb(u8 *buf, int count, void *priv)
{
	struct av7110 *av7110 = (struct av7110 *) priv;
	dprintk(2, "av7110:%p, \n", av7110);

	aux_ring_buffer_write(&av7110->aout, buf, count);
}


#define FREE_COND_TS (dvb_ringbuffer_free(rb) >= 4096)

static ssize_t ts_play(struct av7110 *av7110, const char __user *buf,
		       unsigned long count, int nonblock, int type)
{
	struct dvb_ringbuffer *rb;
	u8 *kb;
	unsigned long todo = count;

	dprintk(2, "%s: type %d cnt %lu\n", __func__, type, count);

	rb = (type) ? &av7110->avout : &av7110->aout;
	kb = av7110->kbuf[type];

	if (!kb)
		return -ENOBUFS;

	if (nonblock && !FREE_COND_TS)
		return -EWOULDBLOCK;

	while (todo >= TS_SIZE) {
		if (!FREE_COND_TS) {
			if (nonblock)
				return count - todo;
			if (wait_event_interruptible(rb->queue, FREE_COND_TS))
				return count - todo;
		}
		if (copy_from_user(kb, buf, TS_SIZE))
			return -EFAULT;
		write_ts_to_decoder(av7110, type, kb, TS_SIZE);
		todo -= TS_SIZE;
		buf += TS_SIZE;
	}

	return count - todo;
}


#define FREE_COND (dvb_ringbuffer_free(&av7110->avout) >= 20 * 1024 && \
		   dvb_ringbuffer_free(&av7110->aout) >= 20 * 1024)

static ssize_t dvb_play(struct av7110 *av7110, const char __user *buf,
			unsigned long count, int nonblock, int type)
{
	unsigned long todo = count, n;
	dprintk(2, "av7110:%p, \n", av7110);

	if (!av7110->kbuf[type])
		return -ENOBUFS;

	if (nonblock && !FREE_COND)
		return -EWOULDBLOCK;

	while (todo > 0) {
		if (!FREE_COND) {
			if (nonblock)
				return count - todo;
			if (wait_event_interruptible(av7110->avout.queue,
						     FREE_COND))
				return count - todo;
		}
		n = todo;
		if (n > IPACKS * 2)
			n = IPACKS * 2;
		if (copy_from_user(av7110->kbuf[type], buf, n))
			return -EFAULT;
		av7110_ipack_instant_repack(av7110->kbuf[type], n,
					    &av7110->ipack[type]);
		todo -= n;
		buf += n;
	}
	return count - todo;
}

static ssize_t dvb_play_kernel(struct av7110 *av7110, const u8 *buf,
			unsigned long count, int nonblock, int type)
{
	unsigned long todo = count, n;
	dprintk(2, "av7110:%p, \n", av7110);

	if (!av7110->kbuf[type])
		return -ENOBUFS;

	if (nonblock && !FREE_COND)
		return -EWOULDBLOCK;

	while (todo > 0) {
		if (!FREE_COND) {
			if (nonblock)
				return count - todo;
			if (wait_event_interruptible(av7110->avout.queue,
						     FREE_COND))
				return count - todo;
		}
		n = todo;
		if (n > IPACKS * 2)
			n = IPACKS * 2;
		av7110_ipack_instant_repack(buf, n, &av7110->ipack[type]);
		todo -= n;
		buf += n;
	}
	return count - todo;
}

static ssize_t dvb_aplay(struct av7110 *av7110, const char __user *buf,
			 unsigned long count, int nonblock, int type)
{
	unsigned long todo = count, n;
	dprintk(2, "av7110:%p, \n", av7110);

	if (!av7110->kbuf[type])
		return -ENOBUFS;
	if (nonblock && dvb_ringbuffer_free(&av7110->aout) < 20 * 1024)
		return -EWOULDBLOCK;

	while (todo > 0) {
		if (dvb_ringbuffer_free(&av7110->aout) < 20 * 1024) {
			if (nonblock)
				return count - todo;
			if (wait_event_interruptible(av7110->aout.queue,
					(dvb_ringbuffer_free(&av7110->aout) >= 20 * 1024)))
				return count-todo;
		}
		n = todo;
		if (n > IPACKS * 2)
			n = IPACKS * 2;
		if (copy_from_user(av7110->kbuf[type], buf, n))
			return -EFAULT;
		av7110_ipack_instant_repack(av7110->kbuf[type], n,
					    &av7110->ipack[type]);
		todo -= n;
		buf += n;
	}
	return count - todo;
}

void av7110_p2t_init(struct av7110_p2t *p, struct dvb_demux_feed *feed)
{
	memset(p->pes, 0, TS_SIZE);
	p->counter = 0;
	p->pos = 0;
	p->frags = 0;
	if (feed)
		p->feed = feed;
}

static void clear_p2t(struct av7110_p2t *p)
{
	memset(p->pes, 0, TS_SIZE);
//	p->counter = 0;
	p->pos = 0;
	p->frags = 0;
}


static int find_pes_header(u8 const *buf, long int length, int *frags)
{
	int c = 0;
	int found = 0;

	*frags = 0;

	while (c < length - 3 && !found) {
		if (buf[c] == 0x00 && buf[c + 1] == 0x00 &&
		    buf[c + 2] == 0x01) {
			switch ( buf[c + 3] ) {
			case PROG_STREAM_MAP:
			case PRIVATE_STREAM2:
			case PROG_STREAM_DIR:
			case ECM_STREAM     :
			case EMM_STREAM     :
			case PADDING_STREAM :
			case DSM_CC_STREAM  :
			case ISO13522_STREAM:
			case PRIVATE_STREAM1:
			case AUDIO_STREAM_S ... AUDIO_STREAM_E:
			case VIDEO_STREAM_S ... VIDEO_STREAM_E:
				found = 1;
				break;

			default:
				c++;
				break;
			}
		} else
			c++;
	}
	if (c == length - 3 && !found) {
		if (buf[length - 1] == 0x00)
			*frags = 1;
		if (buf[length - 2] == 0x00 &&
		    buf[length - 1] == 0x00)
			*frags = 2;
		if (buf[length - 3] == 0x00 &&
		    buf[length - 2] == 0x00 &&
		    buf[length - 1] == 0x01)
			*frags = 3;
		return -1;
	}

	return c;
}

void av7110_p2t_write(u8 const *buf, long int length, u16 pid, struct av7110_p2t *p)
{
	int c, c2, l, add;
	int check, rest;

	c = 0;
	c2 = 0;
	if (p->frags){
		check = 0;
		switch(p->frags) {
		case 1:
			if (buf[c] == 0x00 && buf[c + 1] == 0x01) {
				check = 1;
				c += 2;
			}
			break;
		case 2:
			if (buf[c] == 0x01) {
				check = 1;
				c++;
			}
			break;
		case 3:
			check = 1;
		}
		if (check) {
			switch (buf[c]) {
			case PROG_STREAM_MAP:
			case PRIVATE_STREAM2:
			case PROG_STREAM_DIR:
			case ECM_STREAM     :
			case EMM_STREAM     :
			case PADDING_STREAM :
			case DSM_CC_STREAM  :
			case ISO13522_STREAM:
			case PRIVATE_STREAM1:
			case AUDIO_STREAM_S ... AUDIO_STREAM_E:
			case VIDEO_STREAM_S ... VIDEO_STREAM_E:
				p->pes[0] = 0x00;
				p->pes[1] = 0x00;
				p->pes[2] = 0x01;
				p->pes[3] = buf[c];
				p->pos = 4;
				memcpy(p->pes + p->pos, buf + c, (TS_SIZE - 4) - p->pos);
				c += (TS_SIZE - 4) - p->pos;
				p_to_t(p->pes, (TS_SIZE - 4), pid, &p->counter, p->feed);
				clear_p2t(p);
				break;

			default:
				c = 0;
				break;
			}
		}
		p->frags = 0;
	}

	if (p->pos) {
		c2 = find_pes_header(buf + c, length - c, &p->frags);
		if (c2 >= 0 && c2 < (TS_SIZE - 4) - p->pos)
			l = c2+c;
		else
			l = (TS_SIZE - 4) - p->pos;
		memcpy(p->pes + p->pos, buf, l);
		c += l;
		p->pos += l;
		p_to_t(p->pes, p->pos, pid, &p->counter, p->feed);
		clear_p2t(p);
	}

	add = 0;
	while (c < length) {
		c2 = find_pes_header(buf + c + add, length - c - add, &p->frags);
		if (c2 >= 0) {
			c2 += c + add;
			if (c2 > c){
				p_to_t(buf + c, c2 - c, pid, &p->counter, p->feed);
				c = c2;
				clear_p2t(p);
				add = 0;
			} else
				add = 1;
		} else {
			l = length - c;
			rest = l % (TS_SIZE - 4);
			l -= rest;
			p_to_t(buf + c, l, pid, &p->counter, p->feed);
			memcpy(p->pes, buf + c + l, rest);
			p->pos = rest;
			c = length;
		}
	}
}


static int write_ts_header2(u16 pid, u8 *counter, int pes_start, u8 *buf, u8 length)
{
	int i;
	int c = 0;
	int fill;
	u8 tshead[4] = { 0x47, 0x00, 0x00, 0x10 };

	fill = (TS_SIZE - 4) - length;
	if (pes_start)
		tshead[1] = 0x40;
	if (fill)
		tshead[3] = 0x30;
	tshead[1] |= (u8)((pid & 0x1F00) >> 8);
	tshead[2] |= (u8)(pid & 0x00FF);
	tshead[3] |= ((*counter)++ & 0x0F);
	memcpy(buf, tshead, 4);
	c += 4;

	if (fill) {
		buf[4] = fill - 1;
		c++;
		if (fill > 1) {
			buf[5] = 0x00;
			c++;
		}
		for (i = 6; i < fill + 4; i++) {
			buf[i] = 0xFF;
			c++;
		}
	}

	return c;
}


static void p_to_t(u8 const *buf, long int length, u16 pid, u8 *counter,
		   struct dvb_demux_feed *feed)
{
	int l, pes_start;
	u8 obuf[TS_SIZE];
	long c = 0;

	pes_start = 0;
	if (length > 3 &&
	     buf[0] == 0x00 && buf[1] == 0x00 && buf[2] == 0x01)
		switch (buf[3]) {
			case PROG_STREAM_MAP:
			case PRIVATE_STREAM2:
			case PROG_STREAM_DIR:
			case ECM_STREAM     :
			case EMM_STREAM     :
			case PADDING_STREAM :
			case DSM_CC_STREAM  :
			case ISO13522_STREAM:
			case PRIVATE_STREAM1:
			case AUDIO_STREAM_S ... AUDIO_STREAM_E:
			case VIDEO_STREAM_S ... VIDEO_STREAM_E:
				pes_start = 1;
				break;

			default:
				break;
		}

	while (c < length) {
		memset(obuf, 0, TS_SIZE);
		if (length - c >= (TS_SIZE - 4)){
			l = write_ts_header2(pid, counter, pes_start,
					     obuf, (TS_SIZE - 4));
			memcpy(obuf + l, buf + c, TS_SIZE - l);
			c += TS_SIZE - l;
		} else {
			l = write_ts_header2(pid, counter, pes_start,
					     obuf, length - c);
			memcpy(obuf + l, buf + c, TS_SIZE - l);
			c = length;
		}
		feed->cb.ts(obuf, 188, NULL, 0, &feed->feed.ts, NULL);
		pes_start = 0;
	}
}


static int write_ts_to_decoder(struct av7110 *av7110, int type, const u8 *buf, size_t len)
{
	struct ipack *ipack = &av7110->ipack[type];

	if (buf[1] & TRANS_ERROR) {
		av7110_ipack_reset(ipack);
		return -1;
	}

	if (!(buf[3] & PAYLOAD))
		return -1;

	if (buf[1] & PAY_START)
		av7110_ipack_flush(ipack);

	if (buf[3] & ADAPT_FIELD) {
		len -= buf[4] + 1;
		buf += buf[4] + 1;
		if (!len)
			return 0;
	}

	av7110_ipack_instant_repack(buf + 4, len - 4, ipack);
	return 0;
}


int av7110_write_to_decoder(struct dvb_demux_feed *feed, const u8 *buf, size_t len)
{
	struct dvb_demux *demux = feed->demux;
	struct av7110 *av7110 = (struct av7110 *) demux->priv;

	dprintk(2, "av7110:%p, \n", av7110);

	if (av7110->full_ts && demux->dmx.frontend->source != DMX_MEMORY_FE)
		return 0;

	switch (feed->pes_type) {
	case 0:
		if (av7110->audiostate.stream_source == AUDIO_SOURCE_MEMORY)
			return -EINVAL;
		break;
	case 1:
		if (av7110->videostate.stream_source == VIDEO_SOURCE_MEMORY)
			return -EINVAL;
		break;
	default:
		return -1;
	}

	return write_ts_to_decoder(av7110, feed->pes_type, buf, len);
}



/******************************************************************************
 * Video MPEG decoder events
 ******************************************************************************/
void dvb_video_add_event(struct av7110 *av7110, struct video_event *event)
{
	struct dvb_video_events *events = &av7110->video_events;
	int wp;

	spin_lock_bh(&events->lock);

	wp = (events->eventw + 1) % MAX_VIDEO_EVENT;
	if (wp == events->eventr) {
		events->overflow = 1;
		events->eventr = (events->eventr + 1) % MAX_VIDEO_EVENT;
	}

	//FIXME: timestamp?
	memcpy(&events->events[events->eventw], event, sizeof(struct video_event));
	events->eventw = wp;

	spin_unlock_bh(&events->lock);

	wake_up_interruptible(&events->wait_queue);
}


static int dvb_video_get_event (struct av7110 *av7110, struct video_event *event, int flags)
{
	struct dvb_video_events *events = &av7110->video_events;

	if (events->overflow) {
		events->overflow = 0;
		return -EOVERFLOW;
	}
	if (events->eventw == events->eventr) {
		int ret;

		if (flags & O_NONBLOCK)
			return -EWOULDBLOCK;

		ret = wait_event_interruptible(events->wait_queue,
					       events->eventw != events->eventr);
		if (ret < 0)
			return ret;
	}

	spin_lock_bh(&events->lock);

	memcpy(event, &events->events[events->eventr],
	       sizeof(struct video_event));
	events->eventr = (events->eventr + 1) % MAX_VIDEO_EVENT;

	spin_unlock_bh(&events->lock);

	return 0;
}

/******************************************************************************
 * DVB device file operations
 ******************************************************************************/

static __poll_t dvb_video_poll(struct file *file, poll_table *wait)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	__poll_t mask = 0;

	dprintk(2, "av7110:%p, \n", av7110);

	if ((file->f_flags & O_ACCMODE) != O_RDONLY)
		poll_wait(file, &av7110->avout.queue, wait);

	poll_wait(file, &av7110->video_events.wait_queue, wait);

	if (av7110->video_events.eventw != av7110->video_events.eventr)
		mask = EPOLLPRI;

	if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
		if (av7110->playing) {
			if (FREE_COND)
				mask |= (EPOLLOUT | EPOLLWRNORM);
		} else {
			/* if not playing: may play if asked for */
			mask |= (EPOLLOUT | EPOLLWRNORM);
		}
	}

	return mask;
}

static ssize_t dvb_video_write(struct file *file, const char __user *buf,
			       size_t count, loff_t *ppos)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	unsigned char c;

	dprintk(2, "av7110:%p, \n", av7110);

	if ((file->f_flags & O_ACCMODE) == O_RDONLY)
		return -EPERM;

	if (av7110->videostate.stream_source != VIDEO_SOURCE_MEMORY)
		return -EPERM;

	if (get_user(c, buf))
		return -EFAULT;
	if (c == 0x47 && count % TS_SIZE == 0)
		return ts_play(av7110, buf, count, file->f_flags & O_NONBLOCK, 1);
	else
		return dvb_play(av7110, buf, count, file->f_flags & O_NONBLOCK, 1);
}

static __poll_t dvb_audio_poll(struct file *file, poll_table *wait)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	__poll_t mask = 0;

	dprintk(2, "av7110:%p, \n", av7110);

	poll_wait(file, &av7110->aout.queue, wait);

	if (av7110->playing) {
		if (dvb_ringbuffer_free(&av7110->aout) >= 20 * 1024)
			mask |= (EPOLLOUT | EPOLLWRNORM);
	} else /* if not playing: may play if asked for */
		mask = (EPOLLOUT | EPOLLWRNORM);

	return mask;
}

static ssize_t dvb_audio_write(struct file *file, const char __user *buf,
			       size_t count, loff_t *ppos)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	unsigned char c;

	dprintk(2, "av7110:%p, \n", av7110);

	if (av7110->audiostate.stream_source != AUDIO_SOURCE_MEMORY) {
		printk(KERN_ERR "not audio source memory\n");
		return -EPERM;
	}

	if (get_user(c, buf))
		return -EFAULT;
	if (c == 0x47 && count % TS_SIZE == 0)
		return ts_play(av7110, buf, count, file->f_flags & O_NONBLOCK, 0);
	else
		return dvb_aplay(av7110, buf, count, file->f_flags & O_NONBLOCK, 0);
}

static u8 iframe_header[] = { 0x00, 0x00, 0x01, 0xe0, 0x00, 0x00, 0x80, 0x00, 0x00 };

#define MIN_IFRAME 400000

static int play_iframe(struct av7110 *av7110, char __user *buf, unsigned int len, int nonblock)
{
	unsigned i, n;
	int progressive = 0;
	int match = 0;

	dprintk(2, "av7110:%p, \n", av7110);

	if (len == 0)
		return 0;

	if (!(av7110->playing & RP_VIDEO)) {
		if (av7110_av_start_play(av7110, RP_VIDEO) < 0)
			return -EBUSY;
	}

	/* search in buf for instances of 00 00 01 b5 1? */
	for (i = 0; i < len; i++) {
		unsigned char c;
		if (get_user(c, buf + i))
			return -EFAULT;
		if (match == 5) {
			progressive = c & 0x08;
			match = 0;
		}
		if (c == 0x00) {
			match = (match == 1 || match == 2) ? 2 : 1;
			continue;
		}
		switch (match++) {
		case 2: if (c == 0x01)
				continue;
			break;
		case 3: if (c == 0xb5)
				continue;
			break;
		case 4: if ((c & 0xf0) == 0x10)
				continue;
			break;
		}
		match = 0;
	}

	/* setting n always > 1, fixes problems when playing stillframes
	   consisting of I- and P-Frames */
	n = MIN_IFRAME / len + 1;

	/* FIXME: nonblock? */
	dvb_play_kernel(av7110, iframe_header, sizeof(iframe_header), 0, 1);

	for (i = 0; i < n; i++)
		dvb_play(av7110, buf, len, 0, 1);

	av7110_ipack_flush(&av7110->ipack[1]);

	if (progressive)
		return vidcom(av7110, AV_VIDEO_CMD_FREEZE, 1);
	else
		return 0;
}

#ifdef CONFIG_COMPAT
struct compat_video_still_picture {
	compat_uptr_t iFrame;
	int32_t size;
};
#define VIDEO_STILLPICTURE32 _IOW('o', 30, struct compat_video_still_picture)

struct compat_video_event {
	__s32 type;
	/* unused, make sure to use atomic time for y2038 if it ever gets used */
	compat_long_t timestamp;
	union {
		video_size_t size;
		unsigned int frame_rate;        /* in frames per 1000sec */
		unsigned char vsync_field;      /* unknown/odd/even/progressive */
	} u;
};
#define VIDEO_GET_EVENT32 _IOR('o', 28, struct compat_video_event)

static int dvb_compat_video_get_event(struct av7110 *av7110,
				      struct compat_video_event *event, int flags)
{
	struct video_event ev;
	int ret;

	ret = dvb_video_get_event(av7110, &ev, flags);

	*event = (struct compat_video_event) {
		.type = ev.type,
		.timestamp = ev.timestamp,
		.u.size = ev.u.size,
	};

	return ret;
}
#endif

static int dvb_video_ioctl(struct file *file,
			   unsigned int cmd, void *parg)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	unsigned long arg = (unsigned long) parg;
	int ret = 0;

	dprintk(1, "av7110:%p, cmd=%04x\n", av7110,cmd);

	if ((file->f_flags & O_ACCMODE) == O_RDONLY) {
		if ( cmd != VIDEO_GET_STATUS && cmd != VIDEO_GET_EVENT &&
		     cmd != VIDEO_GET_SIZE ) {
			return -EPERM;
		}
	}

	if (mutex_lock_interruptible(&av7110->ioctl_mutex))
		return -ERESTARTSYS;

	switch (cmd) {
	case VIDEO_STOP:
		av7110->videostate.play_state = VIDEO_STOPPED;
		if (av7110->videostate.stream_source == VIDEO_SOURCE_MEMORY)
			ret = av7110_av_stop(av7110, RP_VIDEO);
		else
			ret = vidcom(av7110, AV_VIDEO_CMD_STOP,
			       av7110->videostate.video_blank ? 0 : 1);
		if (!ret)
			av7110->trickmode = TRICK_NONE;
		break;

	case VIDEO_PLAY:
		av7110->trickmode = TRICK_NONE;
		if (av7110->videostate.play_state == VIDEO_FREEZED) {
			av7110->videostate.play_state = VIDEO_PLAYING;
			ret = vidcom(av7110, AV_VIDEO_CMD_PLAY, 0);
			if (ret)
				break;
		}
		if (av7110->videostate.stream_source == VIDEO_SOURCE_MEMORY) {
			if (av7110->playing == RP_AV) {
				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);
				if (ret)
					break;
				av7110->playing &= ~RP_VIDEO;
			}
			ret = av7110_av_start_play(av7110, RP_VIDEO);
		}
		if (!ret)
			ret = vidcom(av7110, AV_VIDEO_CMD_PLAY, 0);
		if (!ret)
			av7110->videostate.play_state = VIDEO_PLAYING;
		break;

	case VIDEO_FREEZE:
		av7110->videostate.play_state = VIDEO_FREEZED;
		if (av7110->playing & RP_VIDEO)
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Pause, 0);
		else
			ret = vidcom(av7110, AV_VIDEO_CMD_FREEZE, 1);
		if (!ret)
			av7110->trickmode = TRICK_FREEZE;
		break;

	case VIDEO_CONTINUE:
		if (av7110->playing & RP_VIDEO)
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Continue, 0);
		if (!ret)
			ret = vidcom(av7110, AV_VIDEO_CMD_PLAY, 0);
		if (!ret) {
			av7110->videostate.play_state = VIDEO_PLAYING;
			av7110->trickmode = TRICK_NONE;
		}
		break;

	case VIDEO_SELECT_SOURCE:
		av7110->videostate.stream_source = (video_stream_source_t) arg;
		break;

	case VIDEO_SET_BLANK:
		av7110->videostate.video_blank = (int) arg;
		break;

	case VIDEO_GET_STATUS:
		memcpy(parg, &av7110->videostate, sizeof(struct video_status));
		break;

#ifdef CONFIG_COMPAT
	case VIDEO_GET_EVENT32:
		ret = dvb_compat_video_get_event(av7110, parg, file->f_flags);
		break;
#endif

	case VIDEO_GET_EVENT:
		ret = dvb_video_get_event(av7110, parg, file->f_flags);
		break;

	case VIDEO_GET_SIZE:
		memcpy(parg, &av7110->video_size, sizeof(video_size_t));
		break;

	case VIDEO_SET_DISPLAY_FORMAT:
	{
		video_displayformat_t format = (video_displayformat_t) arg;
		switch (format) {
		case VIDEO_PAN_SCAN:
			av7110->display_panscan = VID_PAN_SCAN_PREF;
			break;
		case VIDEO_LETTER_BOX:
			av7110->display_panscan = VID_VC_AND_PS_PREF;
			break;
		case VIDEO_CENTER_CUT_OUT:
			av7110->display_panscan = VID_CENTRE_CUT_PREF;
			break;
		default:
			ret = -EINVAL;
		}
		if (ret < 0)
			break;
		av7110->videostate.display_format = format;
		ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetPanScanType,
				    1, av7110->display_panscan);
		break;
	}

	case VIDEO_SET_FORMAT:
		if (arg > 1) {
			ret = -EINVAL;
			break;
		}
		av7110->display_ar = arg;
		ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetMonitorType,
				    1, (u16) arg);
		break;

#ifdef CONFIG_COMPAT
	case VIDEO_STILLPICTURE32:
	{
		struct compat_video_still_picture *pic =
			(struct compat_video_still_picture *) parg;
		av7110->videostate.stream_source = VIDEO_SOURCE_MEMORY;
		dvb_ringbuffer_flush_spinlock_wakeup(&av7110->avout);
		ret = play_iframe(av7110, compat_ptr(pic->iFrame),
				  pic->size, file->f_flags & O_NONBLOCK);
		break;
	}
#endif

	case VIDEO_STILLPICTURE:
	{
		struct video_still_picture *pic =
			(struct video_still_picture *) parg;
		av7110->videostate.stream_source = VIDEO_SOURCE_MEMORY;
		dvb_ringbuffer_flush_spinlock_wakeup(&av7110->avout);
		ret = play_iframe(av7110, pic->iFrame, pic->size,
				  file->f_flags & O_NONBLOCK);
		break;
	}

	case VIDEO_FAST_FORWARD:
		//note: arg is ignored by firmware
		if (av7110->playing & RP_VIDEO)
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
					    __Scan_I, 2, AV_PES, 0);
		else
			ret = vidcom(av7110, AV_VIDEO_CMD_FFWD, arg);
		if (!ret) {
			av7110->trickmode = TRICK_FAST;
			av7110->videostate.play_state = VIDEO_PLAYING;
		}
		break;

	case VIDEO_SLOWMOTION:
		if (av7110->playing&RP_VIDEO) {
			if (av7110->trickmode != TRICK_SLOW)
				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Slow, 2, 0, 0);
			if (!ret)
				ret = vidcom(av7110, AV_VIDEO_CMD_SLOW, arg);
		} else {
			ret = vidcom(av7110, AV_VIDEO_CMD_PLAY, 0);
			if (!ret)
				ret = vidcom(av7110, AV_VIDEO_CMD_STOP, 0);
			if (!ret)
				ret = vidcom(av7110, AV_VIDEO_CMD_SLOW, arg);
		}
		if (!ret) {
			av7110->trickmode = TRICK_SLOW;
			av7110->videostate.play_state = VIDEO_PLAYING;
		}
		break;

	case VIDEO_GET_CAPABILITIES:
		*(int *)parg = VIDEO_CAP_MPEG1 | VIDEO_CAP_MPEG2 |
			VIDEO_CAP_SYS | VIDEO_CAP_PROG;
		break;

	case VIDEO_CLEAR_BUFFER:
		dvb_ringbuffer_flush_spinlock_wakeup(&av7110->avout);
		av7110_ipack_reset(&av7110->ipack[1]);
		if (av7110->playing == RP_AV) {
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
					    __Play, 2, AV_PES, 0);
			if (ret)
				break;
			if (av7110->trickmode == TRICK_FAST)
				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
						    __Scan_I, 2, AV_PES, 0);
			if (av7110->trickmode == TRICK_SLOW) {
				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
						    __Slow, 2, 0, 0);
				if (!ret)
					ret = vidcom(av7110, AV_VIDEO_CMD_SLOW, arg);
			}
			if (av7110->trickmode == TRICK_FREEZE)
				ret = vidcom(av7110, AV_VIDEO_CMD_STOP, 1);
		}
		break;

	case VIDEO_SET_STREAMTYPE:
		break;

	default:
		ret = -ENOIOCTLCMD;
		break;
	}

	mutex_unlock(&av7110->ioctl_mutex);
	return ret;
}

static int dvb_audio_ioctl(struct file *file,
			   unsigned int cmd, void *parg)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	unsigned long arg = (unsigned long) parg;
	int ret = 0;

	dprintk(1, "av7110:%p, cmd=%04x\n", av7110,cmd);

	if (((file->f_flags & O_ACCMODE) == O_RDONLY) &&
	    (cmd != AUDIO_GET_STATUS))
		return -EPERM;

	if (mutex_lock_interruptible(&av7110->ioctl_mutex))
		return -ERESTARTSYS;

	switch (cmd) {
	case AUDIO_STOP:
		if (av7110->audiostate.stream_source == AUDIO_SOURCE_MEMORY)
			ret = av7110_av_stop(av7110, RP_AUDIO);
		else
			ret = audcom(av7110, AUDIO_CMD_MUTE);
		if (!ret)
			av7110->audiostate.play_state = AUDIO_STOPPED;
		break;

	case AUDIO_PLAY:
		if (av7110->audiostate.stream_source == AUDIO_SOURCE_MEMORY)
			ret = av7110_av_start_play(av7110, RP_AUDIO);
		if (!ret)
			ret = audcom(av7110, AUDIO_CMD_UNMUTE);
		if (!ret)
			av7110->audiostate.play_state = AUDIO_PLAYING;
		break;

	case AUDIO_PAUSE:
		ret = audcom(av7110, AUDIO_CMD_MUTE);
		if (!ret)
			av7110->audiostate.play_state = AUDIO_PAUSED;
		break;

	case AUDIO_CONTINUE:
		if (av7110->audiostate.play_state == AUDIO_PAUSED) {
			av7110->audiostate.play_state = AUDIO_PLAYING;
			ret = audcom(av7110, AUDIO_CMD_UNMUTE | AUDIO_CMD_PCM16);
		}
		break;

	case AUDIO_SELECT_SOURCE:
		av7110->audiostate.stream_source = (audio_stream_source_t) arg;
		break;

	case AUDIO_SET_MUTE:
	{
		ret = audcom(av7110, arg ? AUDIO_CMD_MUTE : AUDIO_CMD_UNMUTE);
		if (!ret)
			av7110->audiostate.mute_state = (int) arg;
		break;
	}

	case AUDIO_SET_AV_SYNC:
		av7110->audiostate.AV_sync_state = (int) arg;
		ret = audcom(av7110, arg ? AUDIO_CMD_SYNC_ON : AUDIO_CMD_SYNC_OFF);
		break;

	case AUDIO_SET_BYPASS_MODE:
		if (FW_VERSION(av7110->arm_app) < 0x2621)
			ret = -EINVAL;
		av7110->audiostate.bypass_mode = (int)arg;
		break;

	case AUDIO_CHANNEL_SELECT:
		av7110->audiostate.channel_select = (audio_channel_select_t) arg;
		switch(av7110->audiostate.channel_select) {
		case AUDIO_STEREO:
			ret = audcom(av7110, AUDIO_CMD_STEREO);
			if (!ret) {
				if (av7110->adac_type == DVB_ADAC_CRYSTAL)
					i2c_writereg(av7110, 0x20, 0x02, 0x49);
				else if (av7110->adac_type == DVB_ADAC_MSP34x5)
					msp_writereg(av7110, MSP_WR_DSP, 0x0008, 0x0220);
			}
			break;
		case AUDIO_MONO_LEFT:
			ret = audcom(av7110, AUDIO_CMD_MONO_L);
			if (!ret) {
				if (av7110->adac_type == DVB_ADAC_CRYSTAL)
					i2c_writereg(av7110, 0x20, 0x02, 0x4a);
				else if (av7110->adac_type == DVB_ADAC_MSP34x5)
					msp_writereg(av7110, MSP_WR_DSP, 0x0008, 0x0200);
			}
			break;
		case AUDIO_MONO_RIGHT:
			ret = audcom(av7110, AUDIO_CMD_MONO_R);
			if (!ret) {
				if (av7110->adac_type == DVB_ADAC_CRYSTAL)
					i2c_writereg(av7110, 0x20, 0x02, 0x45);
				else if (av7110->adac_type == DVB_ADAC_MSP34x5)
					msp_writereg(av7110, MSP_WR_DSP, 0x0008, 0x0210);
			}
			break;
		default:
			ret = -EINVAL;
			break;
		}
		break;

	case AUDIO_GET_STATUS:
		memcpy(parg, &av7110->audiostate, sizeof(struct audio_status));
		break;

	case AUDIO_GET_CAPABILITIES:
		if (FW_VERSION(av7110->arm_app) < 0x2621)
			*(unsigned int *)parg = AUDIO_CAP_LPCM | AUDIO_CAP_MP1 | AUDIO_CAP_MP2;
		else
			*(unsigned int *)parg = AUDIO_CAP_LPCM | AUDIO_CAP_DTS | AUDIO_CAP_AC3 |
						AUDIO_CAP_MP1 | AUDIO_CAP_MP2;
		break;

	case AUDIO_CLEAR_BUFFER:
		dvb_ringbuffer_flush_spinlock_wakeup(&av7110->aout);
		av7110_ipack_reset(&av7110->ipack[0]);
		if (av7110->playing == RP_AV)
			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
					    __Play, 2, AV_PES, 0);
		break;

	case AUDIO_SET_ID:
		break;

	case AUDIO_SET_MIXER:
	{
		struct audio_mixer *amix = (struct audio_mixer *)parg;
		ret = av7110_set_volume(av7110, amix->volume_left, amix->volume_right);
		break;
	}

	case AUDIO_SET_STREAMTYPE:
		break;

	default:
		ret = -ENOIOCTLCMD;
	}

	mutex_unlock(&av7110->ioctl_mutex);
	return ret;
}


static int dvb_video_open(struct inode *inode, struct file *file)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	int err;

	dprintk(2, "av7110:%p, \n", av7110);

	if ((err = dvb_generic_open(inode, file)) < 0)
		return err;

	if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
		dvb_ringbuffer_flush_spinlock_wakeup(&av7110->aout);
		dvb_ringbuffer_flush_spinlock_wakeup(&av7110->avout);
		av7110->video_blank = 1;
		av7110->audiostate.AV_sync_state = 1;
		av7110->videostate.stream_source = VIDEO_SOURCE_DEMUX;

		/*  empty event queue */
		av7110->video_events.eventr = av7110->video_events.eventw = 0;
	}

	return 0;
}

static int dvb_video_release(struct inode *inode, struct file *file)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;

	dprintk(2, "av7110:%p, \n", av7110);

	if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
		av7110_av_stop(av7110, RP_VIDEO);
	}

	return dvb_generic_release(inode, file);
}

static int dvb_audio_open(struct inode *inode, struct file *file)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;
	int err = dvb_generic_open(inode, file);

	dprintk(2, "av7110:%p, \n", av7110);

	if (err < 0)
		return err;
	dvb_ringbuffer_flush_spinlock_wakeup(&av7110->aout);
	av7110->audiostate.stream_source = AUDIO_SOURCE_DEMUX;
	return 0;
}

static int dvb_audio_release(struct inode *inode, struct file *file)
{
	struct dvb_device *dvbdev = file->private_data;
	struct av7110 *av7110 = dvbdev->priv;

	dprintk(2, "av7110:%p, \n", av7110);

	av7110_av_stop(av7110, RP_AUDIO);
	return dvb_generic_release(inode, file);
}



/******************************************************************************
 * driver registration
 ******************************************************************************/

static const struct file_operations dvb_video_fops = {
	.owner		= THIS_MODULE,
	.write		= dvb_video_write,
	.unlocked_ioctl	= dvb_generic_ioctl,
	.compat_ioctl	= dvb_generic_ioctl,
	.open		= dvb_video_open,
	.release	= dvb_video_release,
	.poll		= dvb_video_poll,
	.llseek		= noop_llseek,
};

static struct dvb_device dvbdev_video = {
	.priv		= NULL,
	.users		= 6,
	.readers	= 5,	/* arbitrary */
	.writers	= 1,
	.fops		= &dvb_video_fops,
	.kernel_ioctl	= dvb_video_ioctl,
};

static const struct file_operations dvb_audio_fops = {
	.owner		= THIS_MODULE,
	.write		= dvb_audio_write,
	.unlocked_ioctl	= dvb_generic_ioctl,
	.compat_ioctl	= dvb_generic_ioctl,
	.open		= dvb_audio_open,
	.release	= dvb_audio_release,
	.poll		= dvb_audio_poll,
	.llseek		= noop_llseek,
};

static struct dvb_device dvbdev_audio = {
	.priv		= NULL,
	.users		= 1,
	.writers	= 1,
	.fops		= &dvb_audio_fops,
	.kernel_ioctl	= dvb_audio_ioctl,
};


int av7110_av_register(struct av7110 *av7110)
{
	av7110->audiostate.AV_sync_state = 0;
	av7110->audiostate.mute_state = 0;
	av7110->audiostate.play_state = AUDIO_STOPPED;
	av7110->audiostate.stream_source = AUDIO_SOURCE_DEMUX;
	av7110->audiostate.channel_select = AUDIO_STEREO;
	av7110->audiostate.bypass_mode = 0;

	av7110->videostate.video_blank = 0;
	av7110->videostate.play_state = VIDEO_STOPPED;
	av7110->videostate.stream_source = VIDEO_SOURCE_DEMUX;
	av7110->videostate.video_format = VIDEO_FORMAT_4_3;
	av7110->videostate.display_format = VIDEO_LETTER_BOX;
	av7110->display_ar = VIDEO_FORMAT_4_3;
	av7110->display_panscan = VID_VC_AND_PS_PREF;

	init_waitqueue_head(&av7110->video_events.wait_queue);
	spin_lock_init(&av7110->video_events.lock);
	av7110->video_events.eventw = av7110->video_events.eventr = 0;
	av7110->video_events.overflow = 0;
	memset(&av7110->video_size, 0, sizeof (video_size_t));

	dvb_register_device(&av7110->dvb_adapter, &av7110->video_dev,
			    &dvbdev_video, av7110, DVB_DEVICE_VIDEO, 0);

	dvb_register_device(&av7110->dvb_adapter, &av7110->audio_dev,
			    &dvbdev_audio, av7110, DVB_DEVICE_AUDIO, 0);

	return 0;
}

void av7110_av_unregister(struct av7110 *av7110)
{
	dvb_unregister_device(av7110->audio_dev);
	dvb_unregister_device(av7110->video_dev);
}

int av7110_av_init(struct av7110 *av7110)
{
	void (*play[])(u8 *, int, void *) = { play_audio_cb, play_video_cb };
	int i, ret;

	for (i = 0; i < 2; i++) {
		struct ipack *ipack = av7110->ipack + i;

		ret = av7110_ipack_init(ipack, IPACKS, play[i]);
		if (ret < 0) {
			if (i)
				av7110_ipack_free(--ipack);
			goto out;
		}
		ipack->data = av7110;
	}

	dvb_ringbuffer_init(&av7110->avout, av7110->iobuf, AVOUTLEN);
	dvb_ringbuffer_init(&av7110->aout, av7110->iobuf + AVOUTLEN, AOUTLEN);

	av7110->kbuf[0] = (u8 *)(av7110->iobuf + AVOUTLEN + AOUTLEN + BMPLEN);
	av7110->kbuf[1] = av7110->kbuf[0] + 2 * IPACKS;
out:
	return ret;
}

void av7110_av_exit(struct av7110 *av7110)
{
	av7110_ipack_free(&av7110->ipack[0]);
	av7110_ipack_free(&av7110->ipack[1]);
}
'>7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182 8183 8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427 8428 8429 8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803 8804 8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817 8818 8819 8820 8821 8822 8823 8824 8825 8826 8827 8828 8829 8830 8831 8832 8833 8834 8835 8836 8837 8838 8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863 8864 8865 8866 8867 8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885 8886 8887 8888 8889 8890 8891 8892 8893 8894 8895 8896 8897 8898 8899 8900 8901 8902 8903 8904 8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932 8933 8934 8935 8936 8937 8938 8939 8940 8941 8942 8943 8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959 8960 8961 8962 8963 8964 8965 8966 8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004 9005 9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025 9026 9027 9028 9029 9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051 9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063 9064 9065 9066 9067 9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078 9079 9080 9081 9082 9083 9084 9085 9086 9087 9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098 9099 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176 9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 9205 9206 9207 9208 9209 9210 9211 9212 9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261 9262 9263 9264 9265 9266 9267 9268 9269 9270 9271 9272 9273 9274 9275 9276 9277 9278 9279 9280 9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330 9331 9332 9333 9334 9335 9336 9337 9338 9339 9340 9341 9342 9343 9344 9345 9346 9347 9348 9349 9350 9351 9352 9353 9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421 9422 9423 9424 9425 9426 9427 9428 9429 9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585 9586 9587 9588 9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 9635 9636 9637 9638 9639 9640 9641 9642 9643 9644 9645 9646 9647 9648 9649 9650 9651 9652 9653 9654 9655 9656 9657 9658 9659 9660 9661 9662 9663 9664 9665 9666 9667 9668 9669 9670 9671 9672 9673 9674 9675 9676 9677 9678 9679 9680 9681 9682 9683 9684 9685 9686 9687 9688 9689 9690 9691 9692 9693 9694 9695 9696 9697 9698 9699 9700 9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720 9721 9722 9723 9724 9725 9726 9727 9728 9729 9730 9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746 9747 9748 9749 9750 9751 9752 9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763 9764 9765 9766 9767 9768 9769 9770 9771 9772 9773 9774 9775 9776 9777 9778 9779 9780 9781 9782 9783 9784 9785 9786 9787 9788 9789 9790 9791 9792 9793 9794 9795 9796 9797 9798 9799 9800 9801 9802 9803 9804 9805 9806 9807 9808 9809 9810 9811 9812 9813 9814 9815 9816 9817 9818 9819 9820 9821 9822 9823 9824 9825 9826 9827 9828 9829 9830 9831 9832 9833 9834 9835 9836 9837 9838 9839 9840 9841 9842 9843 9844 9845 9846 9847 9848 9849 9850 9851 9852 9853 9854 9855 9856 9857 9858 9859 9860 9861 9862 9863 9864 9865 9866 9867 9868 9869 9870 9871 9872 9873 9874 9875 9876 9877 9878 9879 9880 9881 9882 9883 9884 9885 9886 9887 9888 9889 9890 9891 9892 9893 9894 9895 9896 9897 9898 9899 9900 9901 9902 9903 9904 9905 9906 9907 9908 9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923 9924 9925 9926 9927 9928 9929 9930 9931 9932 9933 9934 9935 9936 9937 9938 9939 9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962 9963 9964 9965 9966 9967 9968 9969 9970 9971 9972 9973 9974 9975 9976 9977 9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988 9989 9990 9991 9992 9993 9994 9995 9996 9997 9998 9999 10000 10001 10002 10003 10004 10005 10006 10007 10008 10009 10010 10011 10012 10013 10014 10015 10016 10017 10018 10019 10020 10021 10022 10023 10024 10025 10026 10027 10028 10029 10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040 10041 10042 10043 10044 10045 10046 10047 10048 10049 10050 10051 10052 10053 10054 10055 10056 10057 10058 10059 10060 10061 10062 10063 10064 10065 10066 10067 10068 10069 10070 10071 10072 10073 10074 10075 10076 10077 10078 10079 10080 10081 10082 10083 10084 10085 10086 10087 10088 10089 10090 10091 10092 10093 10094 10095 10096 10097 10098 10099 10100 10101 10102 10103 10104 10105 10106 10107 10108 10109 10110 10111 10112 10113 10114 10115 10116 10117 10118 10119 10120 10121 10122 10123 10124 10125 10126 10127 10128 10129 10130 10131 10132 10133 10134 10135 10136 10137 10138 10139 10140 10141 10142 10143 10144 10145 10146 10147 10148 10149 10150 10151 10152 10153 10154 10155 10156 10157 10158 10159 10160 10161 10162 10163 10164 10165 10166 10167 10168 10169 10170 10171 10172 10173 10174 10175 10176 10177 10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204 10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216 10217 10218 10219 10220 10221 10222 10223 10224 10225 10226 10227 10228 10229 10230 10231 10232 10233 10234 10235 10236 10237 10238 10239 10240 10241 10242 10243 10244 10245 10246 10247 10248 10249 10250 10251 10252 10253 10254 10255 10256 10257 10258 10259 10260 10261 10262 10263 10264 10265 10266 10267 10268 10269 10270 10271 10272 10273 10274 10275 10276 10277 10278 10279 10280 10281 10282 10283 10284 10285 10286 10287 10288 10289 10290 10291 10292 10293 10294 10295 10296 10297 10298 10299 10300 10301 10302 10303 10304 10305 10306 10307 10308 10309 10310 10311 10312 10313 10314 10315 10316 10317 10318 10319 10320 10321 10322 10323 10324 10325 10326 10327 10328 10329 10330 10331 10332 10333 10334 10335 10336 10337 10338 10339 10340 10341 10342 10343 10344 10345 10346 10347 10348 10349 10350 10351 10352 10353 10354 10355 10356 10357 10358 10359 10360 10361 10362 10363 10364 10365 10366 10367 10368 10369 10370 10371 10372 10373 10374 10375 10376 10377 10378 10379 10380 10381 10382 10383 10384 10385 10386 10387 10388 10389 10390 10391 10392 10393 10394 10395 10396 10397 10398 10399 10400 10401 10402 10403 10404 10405 10406 10407 10408 10409 10410 10411 10412 10413 10414 10415 10416 10417 10418 10419 10420 10421 10422 10423 10424 10425 10426 10427 10428 10429 10430 10431 10432 10433 10434 10435 10436 10437 10438 10439 10440 10441 10442 10443 10444 10445 10446 10447 10448 10449 10450 10451 10452 10453 10454 10455 10456 10457 10458 10459 10460 10461 10462 10463 10464 10465 10466 10467 10468 10469 10470 10471 10472 10473 10474 10475 10476 10477 10478 10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492 10493 10494 10495 10496 10497 10498 10499 10500 10501 10502 10503 10504 10505 10506 10507 10508 10509 10510 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 10525 10526 10527 10528 10529 10530 10531 10532 10533 10534 10535 10536 10537 10538 10539 10540 10541 10542 10543 10544 10545 10546 10547 10548 10549 10550 10551 10552 10553 10554 10555 10556 10557 10558 10559 10560 10561 10562 10563 10564 10565 10566 10567 10568 10569 10570 10571 10572 10573 10574 10575 10576 10577 10578 10579 10580 10581 10582 10583 10584 10585 10586 10587 10588 10589 10590 10591 10592 10593 10594 10595 10596 10597 10598 10599 10600 10601 10602 10603 10604 10605 10606 10607 10608 10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620 10621 10622 10623 10624 10625 10626 10627 10628 10629 10630 10631 10632 10633 10634 10635 10636 10637 10638 10639 10640 10641 10642 10643 10644 10645 10646 10647 10648 10649 10650 10651 10652 10653 10654 10655 10656 10657 10658 10659 10660 10661 10662 10663 10664 10665 10666 10667 10668 10669 10670 10671 10672 10673 10674 10675 10676 10677 10678 10679 10680 10681 10682 10683 10684 10685 10686 10687 10688 10689 10690 10691 10692 10693 10694 10695 10696 10697 10698 10699 10700 10701 10702 10703 10704 10705 10706 10707 10708 10709 10710 10711 10712 10713 10714 10715 10716 10717 10718 10719 10720 10721 10722 10723 10724 10725 10726 10727 10728 10729 10730 10731 10732 10733 10734 10735 10736 10737 10738 10739 10740 10741 10742 10743 10744 10745 10746 10747 10748 10749 10750 10751 10752 10753 10754 10755 10756 10757 10758 10759 10760 10761 10762 10763 10764 10765 10766 10767 10768 10769 10770 10771 10772 10773 10774 10775 10776 10777 10778 10779 10780 10781 10782 10783 10784 10785 10786 10787 10788 10789 10790 10791 10792 10793 10794 10795 10796 10797 10798 10799 10800 10801 10802 10803 10804 10805 10806 10807 10808 10809 10810 10811 10812 10813 10814 10815 10816 10817 10818 10819 10820 10821 10822 10823 10824 10825 10826 10827 10828 10829 10830 10831 10832 10833 10834 10835 10836 10837 10838 10839 10840 10841 10842 10843 10844 10845 10846 10847 10848 10849 10850 10851 10852 10853 10854 10855 10856 10857 10858 10859 10860 10861 10862 10863 10864 10865 10866 10867 10868 10869 10870 10871 10872 10873 10874 10875 10876 10877 10878 10879 10880 10881 10882 10883 10884 10885 10886 10887 10888 10889 10890 10891 10892 10893 10894 10895 10896 10897 10898 10899 10900 10901 10902 10903 10904 10905 10906 10907 10908 10909 10910 10911 10912 10913 10914 10915 10916 10917 10918 10919 10920 10921 10922 10923 10924 10925 10926 10927 10928 10929 10930 10931 10932 10933 10934 10935 10936 10937 10938 10939 10940 10941 10942 10943 10944 10945 10946 10947 10948 10949 10950 10951 10952 10953 10954 10955 10956 10957 10958 10959 10960 10961 10962 10963 10964 10965 10966 10967 10968 10969 10970 10971 10972 10973 10974 10975 10976 10977 10978 10979 10980 10981 10982 10983 10984 10985 10986 10987 10988 10989 10990 10991 10992 10993 10994 10995 10996 10997 10998 10999 11000 11001 11002 11003 11004 11005 11006 11007 11008 11009 11010 11011 11012 11013 11014 11015 11016 11017 11018 11019 11020 11021 11022 11023 11024 11025 11026 11027 11028 11029 11030 11031 11032 11033 11034 11035 11036 11037 11038 11039 11040 11041 11042 11043 11044 11045 11046 11047 11048 11049 11050 11051 11052 11053 11054 11055 11056 11057 11058 11059 11060 11061 11062 11063 11064 11065 11066 11067 11068 11069 11070 11071 11072 11073 11074 11075 11076 11077 11078 11079 11080 11081 11082 11083 11084 11085 11086 11087 11088 11089 11090 11091 11092 11093 11094 11095 11096 11097 11098 11099 11100 11101 11102 11103 11104 11105 11106 11107 11108 11109 11110 11111 11112 11113 11114 11115 11116 11117 11118 11119 11120 11121 11122 11123 11124 11125 11126 11127 11128 11129 11130 11131 11132 11133 11134 11135 11136 11137 11138 11139 11140 11141 11142 11143 11144 11145 11146 11147 11148 11149 11150 11151 11152 11153 11154 11155 11156 11157 11158 11159 11160 11161 11162 11163 11164 11165 11166 11167 11168 11169 11170 11171 11172 11173 11174 11175 11176 11177 11178 11179 11180 11181 11182 11183 11184 11185 11186 11187 11188 11189 11190 11191 11192 11193 11194 11195 11196 11197 11198 11199 11200 11201 11202 11203 11204 11205 11206 11207 11208 11209 11210 11211 11212 11213 11214 11215 11216 11217 11218 11219 11220 11221 11222 11223 11224 11225 11226 11227 11228 11229 11230 11231 11232 11233 11234 11235 11236 11237 11238 11239 11240 11241 11242 11243 11244 11245 11246 11247 11248 11249 11250 11251 11252 11253 11254 11255 11256 11257 11258 11259 11260 11261 11262 11263 11264 11265 11266 11267 11268 11269 11270 11271 11272 11273 11274 11275 11276 11277 11278 11279 11280 11281 11282 11283 11284 11285 11286 11287 11288 11289 11290 11291 11292 11293 11294 11295 11296 11297 11298 11299 11300 11301 11302 11303 11304 11305 11306 11307 11308 11309 11310 11311 11312 11313 11314 11315 11316 11317 11318 11319 11320 11321 11322 11323 11324 11325 11326 11327 11328 11329 11330 11331 11332 11333 11334 11335 11336 11337 11338 11339 11340 11341 11342 11343 11344 11345 11346 11347 11348 11349 11350 11351 11352 11353 11354 11355 11356 11357 11358 11359 11360 11361 11362 11363 11364 11365 11366 11367 11368 11369 11370 11371 11372 11373 11374 11375 11376 11377 11378 11379 11380 11381 11382 11383 11384 11385 11386 11387 11388 11389 11390 11391 11392 11393 11394 11395 11396 11397 11398 11399 11400 11401 11402 11403 11404 11405 11406 11407 11408 11409 11410 11411 11412 11413 11414 11415 11416 11417 11418 11419 11420 11421 11422 11423 11424 11425 11426 11427 11428 11429 11430 11431 11432 11433 11434 11435 11436 11437 11438 11439 11440 11441 11442 11443 11444 11445 11446 11447 11448 11449 11450 11451 11452 11453 11454 11455 11456 11457 11458 11459 11460 11461 11462 11463 11464 11465 11466 11467 11468 11469 11470 11471 11472 11473 11474 11475 11476 11477 11478 11479 11480 11481 11482 11483 11484 11485 11486 11487 11488 11489 11490 11491 11492 11493 11494 11495 11496 11497 11498 11499 11500 11501 11502 11503 11504 11505 11506 11507 11508 11509 11510 11511 11512 11513 11514 11515 11516 11517 11518 11519 11520 11521 11522 11523 11524 11525 11526 11527 11528 11529 11530 11531 11532 11533 11534 11535 11536 11537 11538 11539 11540 11541 11542 11543 11544 11545 11546 11547 11548 11549 11550 11551 11552 11553 11554 11555 11556 11557 11558 11559 11560 11561 11562 11563 11564 11565 11566 11567 11568 11569 11570 11571 11572 11573 11574 11575 11576 11577 11578 11579 11580 11581 11582 11583 11584 11585 11586 11587 11588 11589 11590 11591 11592 11593 11594 11595 11596 11597 11598 11599 11600 11601 11602 11603 11604 11605 11606 11607 11608 11609 11610 11611 11612 11613 11614 11615 11616 11617 11618 11619 11620 11621 11622 11623 11624 11625 11626 11627 11628 11629 11630 11631 11632 11633 11634 11635 11636 11637 11638 11639 11640 11641 11642 11643 11644 11645 11646 11647 11648 11649 11650 11651 11652 11653 11654 11655 11656 11657 11658 11659 11660 11661 11662 11663 11664 11665 11666 11667 11668 11669 11670 11671 11672 11673 11674 11675 11676 11677 11678 11679 11680 11681 11682 11683 11684 11685 11686 11687 11688 11689 11690 11691 11692 11693 11694 11695 11696 11697 11698 11699 11700 11701 11702 11703 11704 11705 11706 11707 11708 11709 11710 11711 11712 11713 11714 11715 11716 11717 11718 11719 11720 11721 11722 11723 11724 11725 11726 11727 11728 11729 11730 11731 11732 11733 11734 11735 11736 11737 11738 11739 11740 11741 11742 11743 11744 11745 11746 11747 11748 11749 11750 11751 11752 11753 11754 11755 11756 11757 11758 11759 11760 11761 11762 11763 11764 11765 11766 11767 11768 11769 11770 11771 11772 11773 11774 11775 11776 11777 11778 11779 11780 11781 11782 11783 11784 11785 11786 11787 11788 11789 11790 11791 11792 11793 11794 11795 11796 11797 11798 11799 11800 11801 11802 11803 11804 11805 11806 11807 11808 11809 11810 11811 11812 11813 11814 11815 11816 11817 11818 11819 11820 11821 11822 11823 11824 11825 11826 11827 11828 11829 11830 11831 11832 11833 11834 11835 11836 11837 11838 11839 11840 11841 11842 11843 11844 11845 11846 11847 11848 11849 11850 11851 11852 11853 11854 11855 11856 11857 11858 11859 11860 11861 11862 11863 11864 11865 11866 11867 11868 11869 11870 11871 11872 11873 11874 11875 11876 11877 11878 11879 11880 11881 11882 11883 11884 11885 11886 11887 11888 11889 11890 11891 11892 11893 11894 11895 11896 11897 11898 11899 11900 11901 11902 11903 11904 11905 11906 11907 11908 11909 11910 11911 11912 11913 11914 11915 11916 11917 11918 11919 11920 11921 11922 11923 11924 11925 11926 11927 11928 11929 11930 11931 11932 11933 11934 11935 11936 11937 11938 11939 11940 11941 11942 11943 11944 11945 11946 11947 11948 11949 11950 11951 11952 11953 11954 11955 11956 11957 11958 11959 11960 11961 11962 11963 11964 11965 11966 11967 11968 11969 11970 11971 11972 11973 11974 11975 11976 11977 11978 11979 11980 11981 11982 11983 11984 11985 11986 11987 11988 11989 11990 11991 11992 11993 11994 11995 11996 11997 11998 11999 12000 12001 12002 12003 12004 12005 12006 12007 12008 12009 12010 12011 12012 12013 12014 12015 12016 12017 12018 12019 12020 12021 12022 12023 12024 12025 12026 12027 12028 12029 12030 12031 12032 12033 12034 12035 12036 12037 12038 12039 12040 12041 12042 12043 12044 12045 12046 12047 12048 12049 12050 12051 12052 12053 12054 12055 12056 12057 12058 12059 12060 12061 12062 12063 12064 12065 12066 12067 12068 12069 12070 12071 12072 12073 12074 12075 12076 12077 12078 12079 12080 12081 12082 12083 12084 12085 12086 12087 12088 12089 12090 12091 12092 12093 12094 12095 12096 12097 12098 12099 12100 12101 12102 12103 12104 12105 12106 12107 12108 12109 12110 12111 12112 12113 12114 12115 12116 12117 12118 12119 12120 12121 12122 12123 12124 12125 12126 12127 12128 12129 12130 12131 12132 12133 12134 12135 12136 12137 12138 12139 12140 12141 12142 12143 12144 12145 12146 12147 12148 12149 12150 12151 12152 12153 12154 12155 12156 12157 12158 12159 12160 12161 12162 12163 12164 12165 12166 12167 12168 12169 12170 12171 12172 12173 12174 12175 12176 12177 12178 12179 12180 12181 12182 12183 12184 12185 12186 12187 12188 12189 12190 12191 12192 12193 12194 12195 12196 12197 12198 12199 12200 12201 12202 12203 12204 12205 12206 12207 12208 12209 12210 12211 12212 12213 12214 12215 12216 12217 12218 12219 12220 12221 12222 12223 12224 12225 12226 12227 12228 12229 12230 12231 12232 12233 12234 12235 12236 12237 12238 12239 12240 12241 12242 12243 12244 12245 12246 12247 12248 12249 12250 12251 12252 12253 12254 12255 12256 12257 12258 12259 12260 12261 12262 12263 12264 12265 12266 12267 12268 12269 12270 12271 12272 12273 12274 12275 12276 12277 12278 12279 12280 12281 12282 12283 12284 12285 12286 12287 12288 12289 12290 12291 12292 12293 12294 12295 12296 12297 12298 12299 12300 12301 12302 12303 12304 12305 12306 12307 12308 12309 12310 12311 12312 12313 12314 12315 12316 12317 12318 12319 12320 12321 12322 12323 12324 12325 12326 12327 12328 12329 12330 12331 12332 12333 12334 12335 12336 12337 12338 12339 12340 12341 12342 12343 12344 12345 12346 12347 12348 12349 12350 12351 12352 12353 12354 12355 12356 12357 12358 12359 12360 12361 12362 12363 12364 12365 12366 12367 12368 12369 12370 12371 12372 12373 12374 12375 12376 12377 12378 12379 12380 12381 12382 12383 12384 12385 12386 12387 12388 12389 12390 12391 12392 12393 12394 12395 12396 12397 12398 12399 12400 12401 12402 12403 12404 12405 12406 12407 12408 12409 12410 12411 12412 12413 12414 12415 12416 12417 12418 12419 12420 12421 12422 12423 12424 12425 12426 12427 12428 12429 12430 12431 12432 12433 12434 12435 12436 12437 12438 12439 12440 12441 12442 12443 12444 12445 12446 12447 12448 12449 12450 12451 12452 12453 12454 12455 12456 12457 12458 12459 12460 12461 12462 12463 12464 12465 12466 12467 12468 12469 12470 12471 12472 12473 12474 12475 12476 12477 12478 12479 12480 12481 12482 12483 12484 12485 12486 12487 12488 12489 12490 12491 12492 12493 12494 12495 12496 12497 12498 12499 12500 12501 12502 12503 12504 12505 12506 12507 12508 12509 12510 12511 12512 12513 12514 12515 12516 12517 12518 12519 12520 12521 12522 12523 12524 12525 12526 12527 12528 12529 12530 12531 12532 12533 12534 12535 12536 12537 12538 12539 12540 12541 12542 12543 12544 12545 12546 12547 12548 12549 12550 12551 12552 12553 12554 12555 12556 12557 12558 12559 12560 12561 12562 12563 12564 12565 12566 12567 12568 12569 12570 12571 12572 12573 12574 12575 12576 12577 12578 12579 12580 12581 12582 12583 12584 12585 12586 12587 12588 12589 12590 12591 12592 12593 12594 12595 12596 12597 12598 12599 12600 12601 12602 12603 12604 12605 12606 12607 12608 12609 12610 12611 12612 12613 12614 12615 12616 12617 12618 12619 12620 12621 12622 12623 12624 12625 12626 12627 12628 12629 12630 12631 12632 12633 12634 12635 12636 12637 12638 12639 12640 12641 12642 12643 12644 12645 12646 12647 12648 12649 12650 12651 12652 12653 12654 12655 12656 12657 12658 12659 12660 12661 12662 12663 12664 12665 12666 12667 12668 12669 12670 12671 12672 12673 12674 12675 12676 12677 12678 12679 12680 12681 12682 12683 12684 12685 12686 12687 12688 12689 12690 12691 12692 12693 12694 12695 12696 12697 12698 12699 12700 12701 12702 12703 12704 12705 12706 12707 12708 12709 12710 12711 12712 12713 12714 12715 12716 12717 12718 12719 12720 12721 12722 12723 12724 12725 12726 12727 12728 12729 12730 12731 12732 12733 12734 12735 12736 12737 12738 12739 12740 12741 12742 12743 12744 12745 12746 12747 12748 12749 12750 12751 12752 12753 12754 12755 12756 12757 12758 12759 12760 12761 12762 12763 12764 12765 12766 12767 12768 12769 12770 12771 12772 12773 12774 12775 12776 12777 12778 12779 12780 12781 12782 12783 12784 12785 12786 12787 12788 12789 12790 12791 12792 12793 12794 12795 12796 12797 12798 12799 12800 12801 12802 12803 12804 12805 12806 12807 12808 12809 12810 12811 12812 12813 12814 12815 12816 12817 12818 12819 12820 12821 12822 12823 12824 12825 12826 12827 12828 12829 12830 12831 12832 12833 12834 12835 12836 12837 12838 12839 12840 12841 12842 12843 12844 12845 12846 12847 12848 12849 12850 12851 12852 12853 12854 12855 12856 12857 12858 12859 12860 12861 12862 12863 12864 12865 12866 12867 12868 12869 12870 12871 12872 12873 12874 12875 12876 12877 12878 12879 12880 12881 12882 12883 12884 12885 12886 12887 12888 12889 12890 12891 12892 12893 12894 12895 12896 12897 12898 12899 12900 12901 12902 12903 12904 12905 12906 12907 12908 12909 12910 12911 12912 12913 12914 12915 12916 12917 12918 12919 12920 12921 12922 12923 12924 12925 12926 12927 12928 12929 12930 12931 12932 12933 12934 12935 12936 12937 12938 12939 12940 12941 12942 12943 12944 12945 12946 12947 12948 12949 12950 12951 12952 12953 12954 12955 12956 12957 12958 12959 12960 12961 12962 12963 12964 12965 12966 12967 12968 12969 12970 12971 12972 12973 12974 12975 12976 12977 12978 12979 12980 12981 12982 12983 12984 12985 12986 12987 12988 12989 12990 12991 12992 12993 12994 12995 12996 12997 12998 12999 13000 13001 13002 13003 13004 13005 13006 13007 13008 13009 13010 13011 13012 13013 13014 13015 13016 13017 13018 13019 13020 13021 13022 13023 13024 13025 13026 13027 13028 13029 13030 13031 13032 13033 13034 13035 13036 13037 13038 13039 13040 13041 13042 13043 13044 13045 13046 13047 13048 13049 13050 13051 13052 13053 13054 13055 13056 13057 13058 13059 13060 13061 13062 13063 13064 13065 13066 13067 13068 13069 13070 13071 13072 13073 13074 13075 13076 13077 13078 13079 13080 13081 13082 13083 13084 13085 13086 13087 13088 13089 13090 13091 13092 13093 13094 13095 13096 13097 13098 13099 13100 13101 13102 13103 13104 13105 13106 13107 13108 13109 13110 13111 13112 13113 13114 13115 13116 13117 13118 13119 13120 13121 13122 13123 13124 13125 13126 13127 13128 13129 13130 13131 13132 13133 13134 13135 13136 13137 13138 13139 13140 13141 13142 13143 13144 13145 13146 13147 13148 13149 13150 13151 13152 13153 13154 13155 13156 13157 13158 13159 13160 13161 13162 13163 13164 13165 13166 13167 13168 13169 13170 13171 13172 13173 13174 13175 13176 13177 13178 13179 13180 13181 13182 13183 13184 13185 13186 13187 13188 13189 13190 13191 13192 13193 13194 13195 13196 13197 13198 13199 13200 13201 13202 13203 13204 13205 13206 13207 13208 13209 13210 13211 13212 13213 13214 13215 13216 13217 13218 13219 13220 13221 13222 13223 13224 13225 13226 13227 13228 13229 13230 13231 13232 13233 13234 13235 13236 13237 13238 13239 13240 13241 13242 13243 13244 13245 13246 13247 13248 13249 13250 13251 13252 13253 13254 13255 13256 13257 13258 13259 13260 13261 13262 13263 13264 13265 13266 13267 13268 13269 13270 13271 13272 13273 13274 13275 13276 13277 13278 13279 13280 13281 13282 13283 13284 13285 13286 13287 13288 13289 13290 13291 13292 13293 13294 13295 13296 13297 13298 13299 13300 13301 13302 13303 13304 13305 13306 13307 13308 13309 13310 13311 13312 13313 13314 13315 13316 13317 13318 13319 13320 13321 13322 13323 13324 13325 13326 13327 13328 13329 13330 13331 13332 13333 13334 13335 13336 13337 13338 13339 13340 13341 13342 13343 13344 13345 13346 13347 13348 13349 13350 13351 13352 13353 13354 13355 13356 13357 13358 13359 13360 13361 13362 13363 13364 13365 13366 13367 13368 13369 13370 13371 13372 13373 13374 13375 13376 13377 13378 13379 13380 13381 13382 13383 13384 13385 13386 13387 13388 13389 13390 13391 13392 13393 13394 13395 13396 13397 13398 13399 13400 13401 13402 13403 13404 13405 13406 13407 13408 13409 13410 13411 13412 13413 13414 13415 13416 13417 13418 13419 13420 13421 13422 13423 13424 13425 13426 13427 13428 13429 13430 13431 13432 13433 13434 13435 13436 13437 13438 13439 13440 13441 13442 13443 13444 13445 13446 13447 13448 13449 13450 13451 13452 13453 13454 13455 13456 13457 13458 13459 13460 13461 13462 13463 13464 13465 13466 13467 13468 13469 13470 13471 13472 13473 13474 13475 13476 13477 13478 13479 13480 13481 13482 13483 13484 13485 13486 13487 13488 13489 13490 13491 13492 13493 13494 13495 13496 13497 13498 13499 13500 13501 13502 13503 13504 13505 13506 13507 13508 13509 13510 13511 13512 13513 13514 13515 13516 13517 13518 13519 13520 13521 13522 13523 13524 13525 13526 13527 13528 13529 13530 13531 13532 13533 13534 13535 13536 13537 13538 13539 13540 13541 13542 13543 13544 13545 13546 13547 13548 13549 13550 13551 13552 13553 13554 13555 13556 13557 13558 13559 13560 13561 13562 13563 13564 13565 13566 13567 13568 13569 13570 13571 13572 13573 13574 13575 13576 13577 13578 13579 13580 13581 13582 13583 13584 13585 13586 13587 13588 13589 13590 13591 13592 13593 13594 13595 13596 13597 13598 13599 13600 13601 13602 13603 13604 13605 13606 13607 13608 13609 13610 13611 13612 13613 13614 13615 13616 13617 13618 13619 13620 13621 13622 13623 13624 13625 13626 13627 13628 13629 13630 13631 13632 13633 13634 13635 13636 13637 13638 13639 13640 13641 13642 13643 13644 13645 13646 13647 13648 13649 13650 13651 13652 13653 13654 13655 13656 13657 13658 13659 13660 13661 13662 13663 13664 13665 13666 13667 13668 13669 13670 13671 13672 13673 13674 13675 13676 13677 13678 13679 13680 13681 13682 13683 13684 13685 13686 13687 13688 13689 13690 13691 13692 13693 13694 13695 13696 13697 13698 13699 13700 13701 13702 13703 13704 13705 13706 13707 13708 13709 13710 13711 13712 13713 13714 13715 13716 13717 13718 13719 13720 13721 13722 13723 13724 13725 13726 13727 13728 13729 13730 13731 13732 13733 13734 13735 13736 13737 13738 13739 13740 13741 13742 13743 13744 13745 13746 13747 13748 13749 13750 13751 13752 13753 13754 13755 13756 13757 13758 13759 13760 13761 13762 13763 13764 13765 13766 13767 13768 13769 13770 13771 13772 13773 13774 13775 13776 13777 13778 13779 13780 13781 13782 13783 13784 13785 13786 13787 13788 13789 13790 13791 13792 13793 13794 13795 13796 13797 13798 13799 13800 13801 13802 13803 13804 13805 13806 13807 13808 13809 13810 13811 13812 13813 13814 13815 13816 13817 13818 13819 13820 13821 13822 13823 13824 13825 13826 13827 13828 13829 13830 13831 13832 13833 13834 13835 13836 13837 13838 13839 13840 13841 13842 13843 13844 13845 13846 13847 13848 13849 13850 13851 13852 13853 13854 13855 13856 13857 13858 13859 13860 13861 13862 13863 13864 13865 13866 13867 13868 13869 13870 13871 13872 13873 13874 13875 13876 13877 13878 13879 13880 13881 13882 13883 13884 13885 13886 13887 13888 13889 13890 13891 13892 13893 13894 13895 13896 13897 13898 13899 13900 13901 13902 13903 13904 13905 13906 13907 13908 13909 13910 13911 13912 13913 13914 13915 13916 13917 13918 13919 13920 13921 13922 13923 13924 13925 13926 13927 13928 13929 13930 13931 13932 13933 13934 13935 13936 13937 13938 13939 13940 13941 13942 13943 13944 13945 13946 13947 13948 13949 13950 13951 13952 13953 13954 13955 13956 13957 13958 13959 13960 13961 13962 13963 13964 13965 13966 13967 13968 13969 13970 13971 13972 13973 13974 13975 13976 13977 13978 13979 13980 13981 13982 13983 13984 13985 13986 13987 13988 13989 13990 13991 13992 13993 13994 13995 13996 13997 13998 13999 14000 14001 14002 14003 14004 14005 14006 14007 14008 14009 14010 14011 14012 14013 14014 14015 14016 14017 14018 14019 14020 14021 14022 14023 14024 14025 14026 14027 14028 14029 14030 14031 14032 14033 14034 14035 14036 14037 14038 14039 14040 14041 14042 14043 14044 14045 14046 14047 14048 14049 14050 14051 14052 14053 14054 14055 14056 14057 14058 14059 14060 14061 14062 14063 14064 14065 14066 14067 14068 14069 14070 14071 14072 14073 14074 14075 14076 14077 14078 14079 14080 14081 14082 14083 14084 14085 14086 14087 14088 14089 14090 14091 14092 14093 14094 14095 14096 14097 14098 14099 14100 14101 14102 14103 14104 14105 14106 14107 14108 14109 14110 14111 14112 14113 14114 14115 14116 14117 14118 14119 14120 14121 14122 14123 14124 14125 14126 14127 14128 14129 14130 14131 14132 14133 14134 14135 14136 14137 14138 14139 14140 14141 14142 14143 14144 14145 14146 14147 14148 14149 14150 14151 14152 14153 14154 14155 14156 14157 14158 14159 14160 14161 14162 14163 14164 14165 14166 14167 14168 14169 14170 14171 14172 14173 14174 14175 14176 14177 14178 14179 14180 14181 14182 14183 14184 14185 14186 14187 14188 14189 14190 14191 14192 14193 14194 14195 14196 14197 14198 14199 14200 14201 14202 14203 14204 14205 14206 14207 14208 14209 14210 14211 14212 14213 14214 14215 14216 14217 14218 14219 14220 14221 14222 14223 14224 14225 14226 14227 14228 14229 14230 14231 14232 14233 14234 14235 14236 14237 14238 14239 14240 14241 14242 14243 14244 14245 14246 14247 14248 14249 14250 14251 14252 14253 14254 14255 14256 14257 14258 14259 14260 14261 14262 14263 14264 14265 14266 14267 14268 14269 14270 14271 14272 14273 14274 14275 14276 14277 14278 14279 14280 14281 14282 14283 14284 14285 14286 14287 14288 14289 14290 14291 14292 14293 14294 14295 14296 14297 14298 14299 14300 14301 14302 14303 14304 14305 14306 14307 14308 14309 14310 14311 14312 14313 14314 14315 14316 14317 14318 14319 14320 14321 14322 14323 14324 14325 14326 14327 14328 14329 14330 14331 14332 14333 14334 14335 14336 14337 14338 14339 14340 14341 14342 14343 14344 14345 14346 14347 14348 14349 14350 14351 14352 14353 14354 14355 14356 14357 14358 14359 14360 14361 14362 14363 14364 14365 14366 14367 14368 14369 14370 14371 14372 14373 14374 14375 14376 14377 14378 14379 14380 14381 14382 14383 14384 14385 14386 14387 14388 14389 14390 14391 14392 14393 14394 14395 14396 14397 14398 14399 14400 14401 14402 14403 14404 14405 14406 14407 14408 14409 14410 14411 14412 14413 14414 14415 14416 14417 14418 14419 14420 14421 14422 14423 14424 14425 14426 14427 14428 14429 14430 14431 14432 14433 14434 14435 14436 14437 14438 14439 14440 14441 14442 14443 14444 14445 14446 14447 14448 14449 14450 14451 14452 14453 14454 14455 14456 14457 14458 14459 14460 14461 14462 14463 14464 14465 14466 14467 14468 14469 14470 14471 14472 14473 14474 14475 14476 14477 14478 14479 14480 14481 14482 14483 14484 14485 14486 14487 14488 14489 14490 14491 14492 14493 14494 14495 14496 14497 14498 14499 14500 14501 14502 14503 14504 14505 14506 14507 14508 14509 14510 14511 14512 14513 14514 14515 14516 14517 14518 14519 14520 14521 14522 14523 14524 14525 14526 14527 14528 14529 14530 14531 14532 14533 14534 14535 14536 14537 14538 14539 14540 14541 14542 14543 14544 14545 14546 14547 14548 14549 14550 14551 14552 14553 14554 14555 14556 14557 14558 14559 14560 14561 14562 14563 14564 14565 14566 14567 14568 14569 14570 14571 14572 14573 14574 14575 14576 14577 14578 14579 14580 14581 14582 14583 14584 14585 14586 14587 14588 14589 14590 14591 14592 14593 14594 14595 14596 14597 14598 14599 14600 14601 14602 14603 14604 14605 14606 14607 14608 14609 14610 14611 14612 14613 14614 14615 14616 14617 14618 14619 14620 14621 14622 14623 14624 14625 14626 14627 14628 14629 14630 14631 14632 14633 14634 14635 14636 14637 14638 14639 14640 14641 14642 14643 14644 14645 14646 14647 14648 14649 14650 14651 14652 14653 14654 14655 14656 14657 14658 14659 14660 14661 14662 14663 14664 14665 14666 14667 14668 14669 14670 14671 14672 14673 14674 14675 14676 14677 14678 14679 14680 14681 14682 14683 14684 14685 14686 14687 14688 14689 14690 14691 14692 14693 14694 14695 14696 14697 14698 14699 14700 14701 14702 14703 14704 14705 14706 14707 14708 14709 14710 14711 14712 14713 14714 14715 14716 14717 14718 14719 14720 14721 14722 14723 14724 14725 14726 14727 14728 14729 14730 14731 14732 14733 14734 14735 14736 14737 14738 14739 14740 14741 14742 14743 14744 14745 14746 14747 14748 14749 14750 14751 14752 14753 14754 14755 14756 14757 14758 14759 14760 14761 14762 14763 14764 14765 14766 14767 14768 14769 14770 14771 14772 14773 14774 14775 14776 14777 14778 14779 14780 14781 14782 14783 14784 14785 14786 14787 14788 14789 14790 14791 14792 14793 14794 14795 14796 14797 14798 14799 14800 14801 14802 14803 14804 14805 14806 14807 14808 14809 14810 14811 14812 14813 14814 14815 14816 14817 14818 14819 14820 14821 14822 14823 14824 14825 14826 14827 14828 14829 14830 14831 14832 14833 14834 14835 14836 14837 14838 14839 14840 14841 14842 14843 14844 14845 14846 14847 14848 14849 14850 14851 14852 14853 14854 14855 14856 14857 14858 14859 14860 14861 14862 14863 14864 14865 14866 14867 14868 14869 14870 14871 14872 14873 14874 14875 14876 14877 14878 14879 14880 14881 14882 14883 14884 14885 14886 14887 14888 14889 14890 14891 14892 14893 14894 14895 14896 14897 14898 14899 14900 14901 14902 14903 14904 14905 14906 14907 14908 14909 14910 14911 14912 14913 14914 14915 14916 14917 14918 14919 14920 14921 14922 14923 14924 14925 14926 14927 14928 14929 14930 14931 14932 14933 14934 14935 14936 14937 14938 14939 14940 14941 14942 14943 14944 14945 14946 14947 14948 14949 14950 14951 14952 14953 14954 14955 14956 14957 14958 14959 14960 14961 14962 14963 14964 14965 14966 14967 14968 14969 14970 14971 14972 14973 14974 14975 14976 14977 14978 14979 14980 14981 14982 14983 14984 14985 14986 14987 14988 14989 14990 14991 14992 14993 14994 14995 14996 14997 14998 14999 15000 15001 15002 15003 15004 15005 15006 15007 15008 15009 15010 15011 15012 15013 15014 15015 15016 15017 15018 15019 15020 15021 15022 15023 15024 15025 15026 15027 15028 15029 15030 15031 15032 15033 15034 15035 15036 15037 15038 15039 15040 15041 15042 15043 15044 15045 15046 15047 15048 15049 15050 15051 15052 15053 15054 15055 15056 15057 15058 15059 15060 15061 15062 15063 15064 15065 15066 15067 15068 15069 15070 15071 15072 15073 15074 15075 15076 15077 15078 15079 15080 15081 15082 15083 15084 15085 15086 15087 15088 15089 15090 15091 15092 15093 15094 15095 15096 15097 15098 15099 15100 15101 15102 15103 15104 15105 15106 15107 15108 15109 15110 15111 15112 15113 15114 15115 15116 15117 15118 15119 15120 15121 15122 15123 15124 15125 15126 15127 15128 15129 15130 15131 15132 15133 15134 15135 15136 15137 15138 15139 15140 15141 15142 15143 15144 15145 15146 15147 15148 15149 15150 15151 15152 15153 15154 15155 15156 15157 15158 15159 15160 15161 15162 15163 15164 15165 15166 15167 15168 15169 15170 15171 15172 15173 15174 15175 15176 15177 15178 15179 15180 15181 15182 15183 15184 15185 15186 15187 15188 15189 15190 15191 15192 15193 15194 15195 15196 15197 15198 15199 15200 15201 15202 15203 15204 15205 15206 15207 15208 15209 15210 15211 15212 15213 15214 15215 15216 15217 15218 15219 15220 15221 15222 15223 15224 15225 15226 15227 15228 15229 15230 15231 15232 15233 15234 15235 15236 15237 15238 15239 15240 15241 15242 15243 15244 15245 15246 15247 15248 15249 15250 15251 15252 15253 15254 15255 15256 15257 15258 15259 15260 15261 15262 15263 15264 15265 15266 15267 15268 15269 15270 15271 15272 15273 15274 15275 15276 15277 15278 15279 15280 15281 15282 15283 15284 15285 15286 15287 15288 15289 15290 15291 15292 15293 15294 15295 15296 15297 15298 15299 15300 15301 15302 15303 15304 15305 15306 15307 15308 15309 15310 15311 15312 15313 15314 15315 15316 15317 15318 15319 15320 15321 15322 15323 15324 15325 15326 15327 15328 15329 15330 15331 15332 15333 15334 15335 15336 15337 15338 15339 15340 15341 15342 15343 15344 15345 15346 15347 15348 15349 15350 15351 15352 15353 15354 15355 15356 15357 15358 15359 15360 15361 15362 15363 15364 15365 15366 15367 15368 15369 15370 15371 15372 15373 15374 15375 15376 15377 15378 15379 15380 15381 15382 15383 15384 15385 15386 15387 15388 15389 15390 15391 15392 15393 15394 15395 15396 15397 15398 15399 15400 15401 15402 15403 15404 15405 15406 15407 15408 15409 15410 15411 15412 15413 15414 15415 15416 15417 15418 15419 15420 15421 15422 15423 15424 15425 15426 15427 15428 15429 15430 15431 15432 15433 15434 15435 15436 15437 15438 15439 15440 15441 15442 15443 15444 15445 15446 15447 15448 15449 15450 15451 15452 15453 15454 15455 15456 15457 15458 15459 15460 15461 15462 15463 15464 15465 15466 15467 15468 15469 15470 15471 15472 15473 15474 15475 15476 15477 15478 15479 15480 15481 15482 15483 15484 15485 15486 15487 15488 15489 15490 15491 15492 15493 15494 15495 15496 15497 15498 15499 15500 15501 15502 15503 15504 15505 15506 15507 15508 15509 15510 15511 15512 15513 15514 15515 15516 15517 15518 15519 15520 15521 15522 15523 15524 15525 15526 15527 15528 15529 15530 15531 15532 15533 15534 15535 15536 15537 15538 15539 15540 15541 15542 15543 15544 15545 15546 15547 15548 15549 15550 15551 15552 15553 15554 15555 15556 15557 15558 15559 15560 15561 15562 15563 15564 15565 15566 15567 15568 15569 15570 15571 15572 15573 15574 15575 15576 15577 15578 15579 15580 15581 15582 15583 15584 15585 15586 15587 15588 15589 15590 15591 15592 15593 15594 15595 15596 15597 15598 15599 15600 15601 15602 15603 15604 15605 15606 15607 15608 15609 15610 15611 15612 15613 15614 15615 15616 15617 15618 15619 15620 15621 15622 15623 15624 15625 15626 15627 15628 15629 15630 15631 15632 15633 15634 15635 15636 15637 15638 15639 15640 15641 15642 15643 15644 15645 15646 15647 15648 15649 15650 15651 15652 15653 15654 15655 15656 15657 15658 15659 15660 15661 15662 15663 15664 15665 15666 15667 15668 15669 15670 15671 15672 15673 15674 15675 15676 15677 15678 15679 15680 15681 15682 15683 15684 15685 15686 15687 15688 15689 15690 15691 15692 15693 15694 15695 15696 15697 15698 15699 15700 15701 15702 15703 15704 15705 15706 15707 15708 15709 15710 15711 15712 15713 15714 15715 15716 15717 15718 15719 15720 15721 15722 15723 15724 15725 15726 15727 15728 15729 15730 15731 15732 15733 15734 15735 15736 15737 15738 15739 15740 15741 15742 15743 15744 15745 15746 15747 15748 15749 15750 15751 15752 15753 15754 15755 15756 15757 15758 15759 15760 15761 15762 15763 15764 15765 15766 15767 15768 15769 15770 15771 15772 15773 15774 15775 15776 15777 15778 15779 15780 15781 15782 15783 15784 15785 15786 15787 15788 15789 15790 15791 15792 15793 15794 15795 15796 15797 15798 15799 15800 15801 15802 15803 15804 15805 15806 15807 15808 15809 15810 15811 15812 15813 15814 15815 15816 15817 15818 15819 15820 15821 15822 15823 15824 15825 15826 15827 15828 15829 15830 15831 15832 15833 15834 15835 15836 15837 15838 15839 15840 15841 15842 15843 15844 15845 15846 15847 15848 15849 15850 15851 15852 15853 15854 15855 15856 15857 15858 15859 15860 15861 15862 15863 15864 15865 15866 15867 15868 15869 15870 15871 15872 15873 15874 15875 15876 15877 15878 15879 15880 15881 15882 15883 15884 15885 15886 15887 15888 15889 15890 15891 15892 15893 15894 15895 15896 15897 15898 15899 15900 15901 15902 15903 15904 15905 15906 15907 15908 15909 15910 15911 15912 15913 15914 15915 15916 15917 15918 15919 15920 15921 15922 15923 15924 15925 15926 15927 15928 15929 15930 15931 15932 15933 15934 15935 15936 15937 15938 15939 15940 15941 15942 15943 15944 15945 15946 15947 15948 15949 15950 15951 15952 15953 15954 15955 15956 15957 15958 15959 15960 15961 15962 15963 15964 15965 15966 15967 15968 15969 15970 15971 15972 15973 15974 15975 15976 15977 15978 15979 15980 15981 15982 15983 15984 15985 15986 15987 15988 15989 15990 15991 15992 15993 15994 15995 15996 15997 15998 15999 16000 16001 16002 16003 16004 16005 16006 16007 16008 16009 16010 16011 16012 16013 16014 16015 16016 16017 16018 16019 16020 16021 16022 16023 16024 16025 16026 16027 16028 16029 16030 16031 16032 16033 16034 16035 16036 16037 16038 16039 16040 16041 16042 16043 16044 16045 16046 16047 16048 16049 16050 16051 16052 16053 16054 16055 16056 16057 16058 16059 16060 16061 16062 16063 16064 16065 16066 16067 16068 16069 16070 16071 16072 16073 16074 16075 16076 16077 16078 16079 16080 16081 16082 16083 16084 16085 16086 16087 16088 16089 16090 16091 16092 16093 16094 16095 16096 16097 16098 16099 16100 16101 16102 16103 16104 16105 16106 16107 16108 16109 16110 16111 16112 16113 16114 16115 16116 16117 16118 16119 16120 16121 16122 16123 16124 16125 16126 16127 16128 16129 16130 16131 16132 16133 16134 16135 16136 16137 16138 16139 16140 16141 16142 16143 16144 16145 16146 16147 16148 16149 16150 16151 16152 16153 16154 16155 16156 16157 16158 16159 16160 16161 16162 16163 16164 16165 16166 16167 16168 16169 16170 16171 16172 16173 16174 16175 16176 16177 16178 16179 16180 16181 16182 16183 16184 16185 16186 16187 16188 16189 16190 16191 16192 16193 16194 16195 16196 16197 16198 16199 16200 16201 16202 16203 16204 16205 16206 16207 16208 16209 16210 16211 16212 16213 16214 16215 16216 16217 16218 16219 16220 16221 16222 16223 16224 16225 16226 16227 16228 16229 16230 16231 16232 16233 16234 16235 16236 16237 16238 16239 16240 16241 16242 16243 16244 16245 16246 16247 16248 16249 16250 16251 16252 16253 16254 16255 16256 16257 16258 16259 16260 16261 16262 16263 16264 16265 16266 16267 16268 16269 16270 16271 16272 16273 16274 16275 16276 16277 16278 16279 16280 16281 16282 16283 16284 16285 16286 16287 16288 16289 16290 16291 16292 16293 16294 16295 16296 16297 16298 16299 16300 16301 16302 16303 16304 16305 16306 16307 16308 16309 16310 16311 16312 16313 16314 16315 16316 16317 16318 16319 16320 16321 16322 16323 16324 16325 16326 16327 16328 16329 16330 16331 16332 16333 16334 16335 16336 16337 16338 16339 16340 16341 16342 16343 16344 16345 16346 16347 16348 16349 16350 16351 16352 16353 16354 16355 16356 16357 16358 16359 16360 16361 16362 16363 16364 16365 16366 16367 16368 16369 16370 16371 16372 16373 16374 16375 16376 16377 16378 16379 16380 16381 16382 16383 16384 16385 16386 16387 16388 16389 16390 16391 16392 16393 16394 16395 16396 16397 16398 16399 16400 16401 16402 16403 16404 16405 16406 16407 16408 16409 16410 16411 16412 16413 16414 16415 16416 16417 16418 16419 16420 16421 16422 16423 16424 16425 16426 16427 16428 16429 16430 16431 16432 16433 16434 16435 16436 16437 16438 16439 16440 16441 16442 16443 16444 16445 16446 16447 16448 16449 16450 16451 16452 16453 16454 16455 16456 16457 16458 16459 16460 16461 16462 16463 16464 16465 16466 16467 16468 16469 16470 16471 16472 16473 16474 16475 16476 16477 16478 16479 16480 16481 16482 16483 16484 16485 16486 16487 16488 16489 16490 16491 16492 16493 16494 16495 16496 16497 16498 16499 16500 16501 16502 16503 16504 16505 16506 16507 16508 16509 16510 16511 16512 16513 16514 16515 16516 16517 16518 16519 16520 16521 16522 16523 16524 16525 16526 16527 16528 16529 16530 16531 16532 16533 16534 16535 16536 16537 16538 16539 16540 16541 16542 16543 16544 16545 16546 16547 16548 16549 16550 16551 16552 16553 16554 16555 16556 16557 16558 16559 16560 16561 16562 16563 16564 16565 16566 16567 16568 16569 16570 16571 16572 16573 16574 16575 16576 16577 16578 16579 16580 16581 16582 16583 16584 16585 16586 16587 16588 16589 16590 16591 16592 16593 16594 16595 16596 16597 16598 16599 16600 16601 16602 16603 16604 16605 16606 16607 16608 16609 16610 16611 16612 16613 16614 16615 16616 16617 16618 16619 16620 16621 16622 16623 16624 16625 16626 16627 16628 16629 16630 16631 16632 16633 16634 16635 16636 16637 16638 16639 16640 16641 16642 16643 16644 16645 16646 16647 16648 16649 16650 16651 16652 16653 16654 16655 16656 16657 16658 16659 16660 16661 16662 16663 16664 16665 16666 16667 16668 16669 16670 16671 16672 16673 16674 16675 16676 16677 16678 16679 16680 16681 16682 16683 16684 16685 16686 16687 16688 16689 16690 16691 16692 16693 16694 16695 16696 16697 16698 16699 16700 16701 16702 16703 16704 16705 16706 16707 16708 16709 16710 16711 16712 16713 16714 16715 16716 16717 16718 16719 16720 16721 16722 16723 16724 16725 16726 16727 16728 16729 16730 16731 16732 16733 16734 16735 16736 16737 16738 16739 16740 16741 16742 16743 16744 16745 16746 16747 16748 16749 16750 16751 16752 16753 16754 16755 16756 16757 16758 16759 16760 16761 16762 16763 16764 16765 16766 16767 16768 16769 16770 16771 16772 16773 16774 16775 16776 16777 16778 16779 16780 16781 16782 16783 16784 16785 16786 16787 16788 16789 16790 16791 16792 16793 16794 16795 16796 16797 16798 16799 16800 16801 16802 16803 16804 16805 16806 16807 16808 16809 16810 16811 16812 16813 16814 16815 16816 16817 16818 16819 16820 16821 16822 16823 16824 16825 16826 16827 16828 16829 16830 16831 16832 16833 16834 16835 16836 16837 16838 16839 16840 16841 16842 16843 16844 16845 16846 16847 16848 16849 16850 16851 16852 16853 16854 16855 16856 16857 16858 16859 16860 16861 16862 16863 16864 16865 16866 16867 16868 16869 16870 16871 16872 16873 16874 16875 16876 16877 16878 16879 16880 16881 16882 16883 16884 16885 16886 16887 16888 16889 16890 16891 16892 16893 16894 16895 16896 16897 16898 16899 16900 16901 16902 16903 16904 16905 16906 16907 16908 16909 16910 16911 16912 16913 16914 16915 16916 16917 16918 16919 16920 16921 16922 16923 16924 16925 16926 16927 16928 16929 16930 16931 16932 16933 16934 16935 16936 16937 16938 16939 16940 16941 16942 16943 16944 16945 16946 16947 16948 16949 16950 16951 16952 16953 16954 16955 16956 16957 16958 16959 16960 16961 16962 16963 16964 16965 16966 16967 16968 16969 16970 16971 16972 16973 16974 16975 16976 16977 16978 16979 16980 16981 16982 16983 16984 16985 16986 16987 16988 16989 16990 16991 16992 16993 16994 16995 16996 16997 16998 16999 17000 17001 17002 17003 17004 17005 17006 17007 17008 17009 17010 17011 17012 17013 17014 17015 17016 17017 17018 17019 17020 17021 17022 17023 17024 17025 17026 17027 17028 17029 17030 17031 17032 17033 17034 17035 17036 17037 17038 17039 17040 17041 17042 17043 17044 17045 17046 17047 17048 17049 17050 17051 17052 17053 17054 17055 17056 17057 17058 17059 17060 17061 17062 17063 17064 17065 17066 17067 17068 17069 17070 17071 17072 17073 17074 17075 17076 17077 17078 17079 17080 17081 17082 17083 17084 17085 17086 17087 17088 17089 17090 17091 17092 17093 17094 17095 17096 17097 17098 17099 17100 17101 17102 17103 17104 17105 17106 17107 17108 17109 17110 17111 17112 17113 17114 17115 17116 17117 17118 17119 17120 17121 17122 17123 17124 17125 17126 17127 17128 17129 17130 17131 17132 17133 17134 17135 17136 17137 17138 17139 17140 17141 17142 17143 17144 17145 17146 17147 17148 17149 17150 17151 17152 17153 17154 17155 17156 17157 17158 17159 17160 17161 17162 17163 17164 17165 17166 17167 17168 17169 17170 17171 17172 17173 17174 17175 17176 17177 17178 17179 17180 17181 17182 17183 17184 17185 17186 17187 17188 17189 17190 17191 17192 17193 17194 17195 17196 17197 17198 17199 17200 17201 17202 17203 17204 17205 17206 17207 17208 17209 17210 17211 17212 17213 17214 17215 17216 17217 17218 17219 17220 17221 17222 17223 17224 17225 17226 17227 17228 17229 17230 17231 17232 17233 17234 17235 17236 17237 17238 17239 17240 17241 17242 17243 17244 17245 17246 17247 17248 17249 17250 17251 17252 17253 17254 17255 17256 17257 17258 17259 17260 17261 17262 17263 17264 17265 17266 17267 17268 17269 17270 17271 17272 17273 17274 17275 17276 17277 17278 17279 17280 17281 17282 17283 17284 17285 17286 17287 17288 17289 17290 17291 17292 17293 17294 17295 17296 17297 17298 17299 17300 17301 17302 17303 17304 17305 17306 17307 17308 17309 17310 17311 17312 17313 17314 17315 17316 17317 17318 17319 17320 17321 17322 17323 17324 17325 17326 17327 17328 17329 17330 17331 17332 17333 17334 17335 17336 17337 17338 17339 17340 17341 17342 17343 17344 17345 17346 17347 17348 17349 17350 17351 17352 17353 17354 17355 17356 17357 17358 17359 17360 17361 17362 17363 17364 17365 17366 17367 17368 17369 17370 17371 17372 17373 17374 17375 17376 17377 17378 17379 17380 17381 17382 17383 17384 17385 17386 17387 17388 17389 17390 17391 17392 17393 17394 17395 17396 17397 17398 17399 17400 17401 17402 17403 17404 17405 17406 17407 17408 17409 17410 17411 17412 17413 17414 17415 17416 17417 17418 17419 17420 17421 17422 17423 17424 17425 17426 17427 17428 17429 17430 17431 17432 17433 17434 17435 17436 17437 17438 17439 17440 17441 17442 17443 17444 17445 17446 17447 17448 17449 17450 17451 17452 17453 17454 17455 17456 17457 17458 17459 17460 17461 17462 17463 17464 17465 17466 17467 17468 17469 17470 17471 17472 17473 17474 17475 17476 17477 17478 17479 17480 17481 17482 17483 17484 17485 17486 17487 17488 17489 17490 17491 17492 17493 17494 17495 17496 17497 17498 17499 17500 17501 17502 17503 17504 17505 17506 17507 17508 17509 17510 17511 17512 17513 17514 17515 17516 17517 17518 17519 17520 17521 17522 17523 17524 17525 17526 17527 17528 17529 17530 17531 17532 17533 17534 17535 17536 17537 17538 17539 17540 17541 17542 17543 17544 17545 17546 17547 17548 17549 17550 17551 17552 17553 17554 17555 17556 17557 17558 17559 17560 17561 17562 17563 17564 17565 17566 17567 17568 17569 17570 17571 17572 17573 17574 17575 17576 17577 17578 17579 17580 17581 17582 17583 17584 17585 17586 17587 17588 17589 17590 17591 17592 17593 17594 17595 17596 17597 17598 17599 17600 17601 17602 17603 17604 17605 17606 17607 17608 17609 17610 17611 17612 17613 17614 17615 17616 17617 17618 17619 17620 17621 17622 17623 17624 17625 17626 17627 17628 17629 17630 17631 17632 17633 17634 17635 17636 17637 17638 17639 17640 17641 17642 17643 17644 17645 17646 17647 17648 17649 17650 17651 17652 17653 17654 17655 17656 17657 17658 17659 17660 17661 17662 17663 17664 17665 17666 17667 17668 17669 17670 17671 17672 17673 17674 17675 17676 17677 17678 17679 17680 17681 17682 17683 17684 17685 17686 17687 17688 17689 17690 17691 17692 17693 17694 17695 17696 17697 17698 17699 17700 17701 17702 17703 17704 17705 17706 17707 17708 17709 17710 17711 17712 17713 17714 17715 17716 17717 17718 17719 17720 17721 17722 17723 17724 17725 17726 17727 17728 17729 17730 17731 17732 17733 17734 17735 17736 17737 17738 17739 17740 17741 17742 17743 17744 17745 17746 17747 17748 17749 17750 17751 17752 17753 17754 17755 17756 17757 17758 17759 17760 17761 17762 17763 17764 17765 17766 17767 17768 17769 17770 17771 17772 17773 17774 17775 17776 17777 17778 17779 17780 17781 17782 17783 17784 17785 17786 17787 17788 17789 17790 17791 17792 17793 17794 17795 17796 17797 17798 17799 17800 17801 17802 17803 17804 17805 17806 17807 17808 17809 17810 17811 17812 17813 17814 17815 17816 17817 17818 17819 17820 17821 17822 17823 17824 17825 17826 17827 17828 17829 17830 17831 17832 17833 17834 17835 17836 17837 17838 17839 17840 17841 17842 17843 17844 17845 17846 17847 17848 17849 17850 17851 17852 17853 17854 17855 17856 17857 17858 17859 17860 17861 17862 17863 17864 17865 17866 17867 17868 17869 17870 17871 17872 17873 17874 17875 17876 17877 17878 17879 17880 17881 17882 17883 17884 17885 17886 17887 17888 17889 17890 17891 17892 17893 17894 17895 17896 17897 17898 17899 17900 17901 17902 17903 17904 17905 17906 17907 17908 17909 17910 17911 17912 17913 17914 17915 17916 17917 17918 17919 17920 17921 17922 17923 17924 17925 17926 17927 17928 17929 17930 17931 17932 17933 17934 17935 17936 17937 17938 17939 17940 17941 17942 17943 17944 17945 17946 17947 17948 17949 17950 17951 17952 17953 17954 17955 17956 17957 17958 17959 17960 17961 17962 17963 17964 17965 17966 17967 17968 17969 17970 17971 17972 17973 17974 17975 17976 17977 17978 17979 17980 17981 17982 17983 17984 17985 17986 17987 17988 17989 17990 17991 17992 17993 17994 17995 17996 17997 17998 17999 18000 18001 18002 18003 18004 18005 18006 18007 18008 18009 18010 18011 18012 18013 18014 18015 18016 18017 18018 18019 18020 18021 18022 18023 18024 18025 18026 18027 18028 18029 18030 18031 18032 18033 18034 18035 18036 18037 18038 18039 18040 18041 18042 18043 18044 18045 18046 18047 18048 18049 18050 18051 18052 18053 18054 18055 18056 18057 18058 18059 18060 18061 18062 18063 18064 18065 18066 18067 18068 18069 18070 18071 18072 18073 18074 18075 18076 18077 18078 18079 18080 18081 18082 18083 18084 18085 18086 18087 18088 18089 18090 18091 18092 18093 18094 18095 18096 18097 18098 18099 18100 18101 18102 18103 18104 18105 18106 18107 18108 18109 18110 18111 18112 18113 18114 18115 18116 18117 18118 18119 18120 18121 18122 18123 18124 18125 18126 18127 18128 18129 18130 18131 18132 18133 18134 18135 18136 18137 18138 18139 18140 18141 18142 18143 18144 18145 18146 18147 18148 18149 18150 18151 18152 18153 18154 18155 18156 18157 18158 18159 18160 18161 18162 18163 18164 18165 18166 18167 18168 18169 18170 18171 18172 18173 18174 18175 18176 18177 18178 18179 18180 18181 18182 18183 18184 18185 18186 18187 18188 18189 18190 18191 18192 18193 18194 18195 18196 18197 18198 18199 18200 18201 18202 18203 18204 18205 18206 18207 18208 18209 18210 18211 18212 18213 18214 18215 18216 18217 18218 18219 18220 18221 18222 18223 18224 18225 18226 18227 18228 18229 18230 18231 18232 18233 18234 18235 18236 18237 18238 18239 18240 18241 18242 18243 18244 18245 18246 18247 18248 18249 18250 18251 18252 18253 18254 18255 18256 18257 18258 18259 18260 18261 18262 18263 18264 18265 18266 18267 18268 18269 18270 18271 18272 18273 18274 18275 18276 18277 18278 18279 18280 18281 18282 18283 18284 18285 18286 18287 18288 18289 18290 18291 18292 18293 18294 18295 18296 18297 18298 18299 18300 18301 18302 18303 18304 18305 18306 18307 18308 18309 18310 18311 18312 18313 18314 18315 18316 18317 18318 18319 18320 18321 18322 18323 18324 18325 18326 18327 18328 18329 18330 18331 18332 18333 18334 18335 18336 18337 18338 18339 18340 18341 18342 18343 18344 18345 18346 18347 18348 18349 18350 18351 18352 18353 18354 18355 18356 18357 18358 18359 18360 18361 18362 18363 18364 18365 18366 18367 18368 18369 18370 18371 18372 18373 18374 18375 18376 18377 18378 18379 18380 18381 18382 18383 18384 18385 18386 18387 18388 18389 18390 18391 18392 18393 18394 18395 18396 18397 18398 18399 18400 18401 18402 18403 18404 18405 18406 18407 18408 18409 18410 18411 18412 18413 18414 18415 18416 18417 18418 18419 18420 18421 18422 18423 18424 18425 18426 18427 18428 18429 18430 18431 18432 18433 18434 18435 18436 18437 18438 18439 18440 18441 18442 18443 18444 18445 18446 18447 18448 18449 18450 18451 18452 18453 18454 18455 18456 18457 18458 18459 18460 18461 18462 18463 18464 18465 18466 18467 18468 18469 18470 18471 18472 18473 18474 18475 18476 18477 18478 18479 18480 18481 18482 18483 18484 18485 18486 18487 18488 18489 18490 18491 18492 18493 18494 18495 18496 18497 18498 18499 18500 18501 18502 18503 18504 18505 18506 18507 18508 18509 18510 18511 18512 18513 18514 18515 18516 18517 18518 18519 18520 18521 18522 18523 18524 18525 18526 18527 18528 18529 18530 18531 18532 18533 18534 18535 18536 18537 18538 18539 18540 18541 18542 18543 18544 18545 18546 18547 18548 18549 18550 18551 18552 18553 18554 18555 18556 18557 18558 18559 18560 18561 18562 18563 18564 18565 18566 18567 18568 18569 18570 18571 18572 18573 18574 18575 18576 18577 18578 18579 18580 18581 18582 18583 18584 18585 18586 18587 18588 18589 18590 18591 18592 18593 18594 18595 18596 18597 18598 18599 18600 18601 18602 18603 18604 18605 18606 18607 18608 18609 18610 18611 18612 18613 18614 18615 18616 18617 18618 18619 18620 18621 18622 18623 18624 18625 18626 18627 18628 18629 18630 18631 18632 18633 18634 18635 18636 18637 18638 18639 18640 18641 18642 18643 18644 18645 18646 18647 18648 18649 18650 18651 18652 18653 18654 18655 18656 18657 18658 18659 18660 18661 18662 18663 18664 18665 18666 18667 18668 18669 18670 18671 18672 18673 18674 18675 18676 18677 18678 18679 18680 18681 18682 18683 18684 18685 18686 18687 18688 18689 18690 18691 18692 18693 18694 18695 18696 18697 18698 18699 18700 18701 18702 18703 18704 18705 18706 18707 18708 18709 18710 18711 18712 18713 18714 18715 18716 18717 18718 18719 18720 18721 18722 18723 18724 18725 18726 18727 18728 18729 18730 18731 18732 18733 18734 18735 18736 18737 18738 18739 18740 18741 18742 18743 18744 18745 18746 18747 18748 18749 18750 18751 18752 18753 18754 18755 18756 18757 18758 18759 18760 18761 18762 18763 18764 18765 18766 18767 18768 18769 18770 18771 18772 18773 18774 18775 18776 18777 18778 18779 18780 18781 18782 18783 18784 18785 18786 18787 18788 18789 18790 18791 18792 18793 18794 18795 18796 18797 18798 18799 18800 18801 18802 18803 18804 18805 18806 18807 18808 18809 18810 18811 18812 18813 18814 18815 18816 18817 18818 18819 18820 18821 18822 18823 18824 18825 18826 18827 18828 18829 18830 18831 18832 18833 18834 18835 18836 18837 18838 18839 18840 18841 18842 18843 18844 18845 18846 18847 18848 18849 18850 18851 18852 18853 18854 18855 18856 18857 18858 18859 18860 18861 18862 18863 18864 18865 18866 18867 18868 18869 18870 18871 18872 18873 18874 18875 18876 18877 18878 18879 18880 18881 18882 18883 18884 18885 18886 18887 18888 18889 18890 18891 18892 18893 18894 18895 18896 18897 18898 18899 18900 18901 18902 18903 18904 18905 18906 18907 18908 18909 18910 18911 18912 18913 18914 18915 18916 18917 18918 18919 18920 18921 18922 18923 18924 18925 18926 18927 18928 18929 18930 18931 18932 18933 18934 18935 18936 18937 18938 18939 18940 18941 18942 18943 18944 18945 18946 18947 18948 18949 18950 18951 18952 18953 18954 18955 18956 18957 18958 18959 18960 18961 18962 18963 18964 18965 18966 18967 18968 18969 18970 18971 18972 18973 18974 18975 18976 18977 18978 18979 18980 18981 18982 18983 18984 18985 18986 18987 18988 18989 18990 18991 18992 18993 18994 18995 18996 18997 18998 18999 19000 19001 19002 19003 19004 19005 19006 19007 19008 19009 19010 19011 19012 19013 19014 19015 19016 19017 19018 19019 19020 19021 19022 19023 19024 19025 19026 19027 19028 19029 19030 19031 19032 19033 19034 19035 19036 19037 19038 19039 19040 19041 19042 19043 19044 19045 19046 19047 19048 19049 19050 19051 19052 19053 19054 19055 19056 19057 19058 19059 19060 19061 19062 19063 19064 19065 19066 19067 19068 19069 19070 19071 19072 19073 19074 19075 19076 19077 19078 19079 19080 19081 19082 19083 19084 19085 19086 19087 19088 19089 19090 19091 19092 19093 19094 19095 19096 19097 19098 19099 19100 19101 19102 19103 19104 19105 19106 19107 19108 19109 19110 19111 19112 19113 19114 19115 19116 19117 19118 19119 19120 19121 19122 19123 19124 19125 19126 19127 19128 19129 19130 19131 19132 19133 19134 19135 19136 19137 19138 19139 19140 19141 19142 19143 19144 19145 19146 19147 19148 19149 19150 19151 19152 19153 19154 19155 19156 19157 19158 19159 19160 19161 19162 19163 19164 19165 19166 19167 19168 19169 19170 19171 19172 19173 19174 19175 19176 19177 19178 19179 19180 19181 19182 19183 19184 19185 19186 19187 19188 19189 19190 19191 19192 19193 19194 19195 19196 19197 19198 19199 19200 19201 19202 19203 19204 19205 19206 19207 19208 19209 19210 19211 19212 19213 19214 19215 19216 19217 19218 19219 19220 19221 19222 19223 19224 19225 19226 19227 19228 19229 19230 19231 19232 19233 19234 19235 19236 19237 19238 19239 19240 19241 19242 19243 19244 19245 19246 19247 19248 19249 19250 19251 19252 19253 19254 19255 19256 19257 19258 19259 19260 19261 19262 19263 19264 19265 19266 19267 19268 19269 19270 19271 19272 19273 19274 19275 19276 19277 19278 19279 19280 19281 19282 19283 19284 19285 19286 19287 19288 19289 19290 19291 19292 19293 19294 19295 19296 19297 19298 19299 19300 19301 19302 19303 19304 19305 19306 19307 19308 19309 19310 19311 19312 19313 19314 19315 19316 19317 19318 19319 19320 19321 19322 19323 19324 19325 19326 19327 19328 19329 19330 19331 19332 19333 19334 19335 19336 19337 19338 19339 19340 19341 19342 19343 19344 19345 19346 19347 19348 19349 19350 19351 19352 19353 19354 19355 19356 19357 19358 19359 19360 19361 19362 19363 19364 19365 19366 19367 19368 19369 19370 19371 19372 19373 19374 19375 19376 19377 19378 19379 19380 19381 19382 19383 19384 19385 19386 19387 19388 19389 19390 19391 19392 19393 19394 19395 19396 19397 19398 19399 19400 19401 19402 19403 19404 19405 19406 19407 19408 19409 19410 19411 19412 19413 19414 19415 19416 19417 19418 19419 19420 19421 19422 19423 19424 19425 19426 19427 19428 19429 19430 19431 19432 19433 19434 19435 19436 19437 19438 19439 19440 19441 19442 19443 19444 19445 19446 19447 19448 19449 19450 19451 19452 19453 19454 19455 19456 19457 19458 19459 19460 19461 19462 19463 19464 19465 19466 19467 19468 19469 19470 19471 19472 19473 19474 19475 19476 19477 19478 19479 19480 19481 19482 19483 19484 19485 19486 19487 19488 19489 19490 19491 19492 19493 19494 19495 19496 19497 19498 19499 19500 19501 19502 19503 19504 19505 19506 19507 19508 19509 19510 19511 19512 19513 19514 19515 19516 19517 19518 19519 19520 19521 19522 19523 19524 19525 19526 19527 19528 19529 19530 19531 19532 19533 19534 19535 19536 19537 19538 19539 19540 19541 19542 19543 19544 19545 19546 19547 19548 19549 19550 19551 19552 19553 19554 19555 19556 19557 19558 19559 19560 19561 19562 19563 19564 19565 19566 19567 19568 19569 19570 19571 19572 19573 19574 19575 19576 19577 19578 19579 19580 19581 19582 19583 19584 19585 19586 19587 19588 19589 19590 19591 19592 19593 19594 19595 19596 19597 19598 19599 19600 19601 19602 19603 19604 19605 19606 19607 19608 19609 19610 19611 19612 19613 19614 19615 19616 19617 19618 19619 19620 19621 19622 19623 19624 19625 19626 19627 19628 19629 19630 19631 19632 19633 19634 19635 19636 19637 19638 19639 19640 19641 19642 19643 19644 19645 19646 19647 19648 19649 19650 19651 19652 19653 19654 19655 19656 19657 19658 19659 19660 19661 19662 19663 19664 19665 19666 19667 19668 19669 19670 19671 19672 19673 19674 19675 19676 19677 19678 19679 19680 19681 19682 19683 19684 19685 19686 19687 19688 19689 19690 19691 19692 19693 19694 19695 19696 19697 19698 19699 19700 19701 19702 19703 19704 19705 19706 19707 19708 19709 19710 19711 19712 19713 19714 19715 19716 19717 19718 19719 19720 19721 19722 19723 19724 19725 19726 19727 19728 19729 19730 19731 19732 19733 19734 19735 19736 19737 19738 19739 19740 19741 19742 19743 19744 19745 19746 19747 19748 19749 19750 19751 19752 19753 19754 19755 19756 19757 19758 19759 19760 19761 19762 19763 19764 19765 19766 19767 19768 19769 19770 19771 19772 19773 19774 19775 19776 19777 19778 19779 19780 19781 19782 19783 19784 19785 19786 19787 19788 19789 19790 19791 19792 19793 19794 19795 19796 19797 19798 19799 19800 19801 19802 19803 19804 19805 19806 19807 19808 19809 19810 19811 19812 19813 19814 19815 19816 19817 19818 19819 19820 19821 19822 19823 19824 19825 19826 19827 19828 19829 19830 19831 19832 19833 19834 19835 19836 19837 19838 19839 19840 19841 19842 19843 19844 19845 19846 19847 19848 19849 19850 19851 19852 19853 19854 19855 19856 19857 19858 19859 19860 19861 19862 19863 19864 19865 19866 19867 19868 19869 19870 19871 19872 19873 19874 19875 19876 19877 19878 19879 19880 19881 19882 19883 19884 19885 19886 19887 19888 19889 19890 19891 19892 19893 19894 19895 19896 19897 19898 19899 19900 19901 19902 19903 19904 19905 19906 19907 19908 19909 19910 19911 19912 19913 19914 19915 19916 19917 19918 19919 19920 19921 19922 19923 19924 19925 19926 19927 19928 19929 19930 19931 19932 19933 19934 19935 19936 19937 19938 19939 19940 19941 19942 19943 19944 19945 19946 19947 19948 19949 19950 19951 19952 19953 19954 19955 19956 19957 19958 19959 19960 19961 19962 19963 19964 19965 19966 19967 19968 19969 19970 19971 19972 19973 19974 19975 19976 19977 19978 19979 19980 19981 19982 19983 19984 19985 19986 19987 19988 19989 19990 19991 19992 19993 19994 19995 19996 19997 19998 19999 20000 20001 20002 20003 20004 20005 20006 20007 20008 20009 20010 20011 20012 20013 20014 20015 20016 20017 20018 20019 20020 20021 20022 20023 20024 20025 20026 20027 20028 20029 20030 20031 20032 20033 20034 20035 20036 20037 20038 20039 20040 20041 20042 20043 20044 20045 20046 20047 20048 20049 20050 20051 20052 20053 20054 20055 20056 20057 20058 20059 20060 20061 20062 20063 20064 20065 20066 20067 20068 20069 20070 20071 20072 20073 20074 20075 20076 20077 20078 20079 20080 20081 20082 20083 20084 20085 20086 20087 20088 20089 20090 20091 20092 20093 20094 20095 20096 20097 20098 20099 20100 20101 20102 20103 20104 20105 20106 20107 20108 20109 20110 20111 20112 20113 20114 20115 20116 20117 20118 20119 20120 20121 20122 20123 20124 20125 20126 20127 20128 20129 20130 20131 20132 20133 20134 20135 20136 20137 20138 20139 20140 20141 20142 20143 20144 20145 20146 20147 20148 20149 20150 20151 20152 20153 20154 20155 20156 20157 20158 20159 20160 20161 20162 20163 20164 20165 20166 20167 20168 20169 20170 20171 20172 20173 20174 20175 20176 20177 20178 20179 20180 20181 20182 20183 20184 20185 20186 20187 20188 20189 20190 20191 20192 20193 20194 20195 20196 20197 20198 20199 20200 20201 20202 20203 20204 20205 20206 20207 20208 20209 20210 20211 20212 20213 20214 20215 20216 20217 20218 20219 20220 20221 20222 20223 20224 20225 20226 20227 20228 20229 20230 20231 20232 20233 20234 20235 20236 20237 20238 20239 20240 20241 20242 20243 20244 20245 20246 20247 20248 20249 20250 20251 20252 20253 20254 20255 20256 20257 20258 20259 20260 20261 20262 20263 20264 20265 20266 20267 20268 20269 20270 20271 20272 20273 20274 20275 20276 20277 20278 20279 20280 20281 20282 20283 20284 20285 20286 20287 20288 20289 20290 20291 20292 20293 20294 20295 20296 20297 20298 20299 20300 20301 20302 20303 20304 20305 20306 20307 20308 20309 20310 20311 20312 20313 20314 20315 20316 20317 20318 20319 20320 20321 20322 20323 20324 20325 20326 20327 20328 20329 20330 20331 20332 20333 20334 20335 20336 20337 20338 20339 20340 20341 20342 20343 20344 20345 20346 20347 20348 20349 20350 20351 20352 20353 20354 20355 20356 20357 20358 20359 20360 20361 20362 20363 20364 20365 20366 20367 20368 20369 20370 20371 20372 20373 20374 20375 20376 20377 20378 20379 20380 20381 20382 20383 20384 20385 20386 20387 20388 20389
/*******************************************************************
 * This file is part of the Emulex Linux Device Driver for         *
 * Fibre Channel Host Bus Adapters.                                *
 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term *
 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
 * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
 * EMULEX and SLI are trademarks of Emulex.                        *
 * www.broadcom.com                                                *
 * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
 *                                                                 *
 * This program is free software; you can redistribute it and/or   *
 * modify it under the terms of version 2 of the GNU General       *
 * Public License as published by the Free Software Foundation.    *
 * This program is distributed in the hope that it will be useful. *
 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
 * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
 * more details, a copy of which can be found in the file COPYING  *
 * included with this package.                                     *
 *******************************************************************/

#include <linux/blkdev.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/lockdep.h>

#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport_fc.h>
#include <scsi/fc/fc_fs.h>
#include <linux/aer.h>
#ifdef CONFIG_X86
#include <asm/set_memory.h>
#endif

#include <linux/nvme-fc-driver.h>

#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli.h"
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
#include "lpfc.h"
#include "lpfc_scsi.h"
#include "lpfc_nvme.h"
#include "lpfc_nvmet.h"
#include "lpfc_crtn.h"
#include "lpfc_logmsg.h"
#include "lpfc_compat.h"
#include "lpfc_debugfs.h"
#include "lpfc_vport.h"
#include "lpfc_version.h"

/* There are only four IOCB completion types. */
typedef enum _lpfc_iocb_type {
	LPFC_UNKNOWN_IOCB,
	LPFC_UNSOL_IOCB,
	LPFC_SOL_IOCB,
	LPFC_ABORT_IOCB
} lpfc_iocb_type;


/* Provide function prototypes local to this module. */
static int lpfc_sli_issue_mbox_s4(struct lpfc_hba *, LPFC_MBOXQ_t *,
				  uint32_t);
static int lpfc_sli4_read_rev(struct lpfc_hba *, LPFC_MBOXQ_t *,
			      uint8_t *, uint32_t *);
static struct lpfc_iocbq *lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *,
							 struct lpfc_iocbq *);
static void lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *,
				      struct hbq_dmabuf *);
static void lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport,
					  struct hbq_dmabuf *dmabuf);
static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba,
				   struct lpfc_queue *cq, struct lpfc_cqe *cqe);
static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *,
				       int);
static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba,
				     struct lpfc_queue *eq,
				     struct lpfc_eqe *eqe);
static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba);
static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba);

static IOCB_t *
lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
{
	return &iocbq->iocb;
}

#if defined(CONFIG_64BIT) && defined(__LITTLE_ENDIAN)
/**
 * lpfc_sli4_pcimem_bcopy - SLI4 memory copy function
 * @srcp: Source memory pointer.
 * @destp: Destination memory pointer.
 * @cnt: Number of words required to be copied.
 *       Must be a multiple of sizeof(uint64_t)
 *
 * This function is used for copying data between driver memory
 * and the SLI WQ. This function also changes the endianness
 * of each word if native endianness is different from SLI
 * endianness. This function can be called with or without
 * lock.
 **/
void
lpfc_sli4_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
{
	uint64_t *src = srcp;
	uint64_t *dest = destp;
	int i;

	for (i = 0; i < (int)cnt; i += sizeof(uint64_t))
		*dest++ = *src++;
}
#else
#define lpfc_sli4_pcimem_bcopy(a, b, c) lpfc_sli_pcimem_bcopy(a, b, c)
#endif

/**
 * lpfc_sli4_wq_put - Put a Work Queue Entry on an Work Queue
 * @q: The Work Queue to operate on.
 * @wqe: The work Queue Entry to put on the Work queue.
 *
 * This routine will copy the contents of @wqe to the next available entry on
 * the @q. This function will then ring the Work Queue Doorbell to signal the
 * HBA to start processing the Work Queue Entry. This function returns 0 if
 * successful. If no entries are available on @q then this function will return
 * -ENOMEM.
 * The caller is expected to hold the hbalock when calling this routine.
 **/
static int
lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
{
	union lpfc_wqe *temp_wqe;
	struct lpfc_register doorbell;
	uint32_t host_index;
	uint32_t idx;
	uint32_t i = 0;
	uint8_t *tmp;
	u32 if_type;

	/* sanity check on queue memory */
	if (unlikely(!q))
		return -ENOMEM;
	temp_wqe = lpfc_sli4_qe(q, q->host_index);

	/* If the host has not yet processed the next entry then we are done */
	idx = ((q->host_index + 1) % q->entry_count);
	if (idx == q->hba_index) {
		q->WQ_overflow++;
		return -EBUSY;
	}
	q->WQ_posted++;
	/* set consumption flag every once in a while */
	if (!((q->host_index + 1) % q->notify_interval))
		bf_set(wqe_wqec, &wqe->generic.wqe_com, 1);
	else
		bf_set(wqe_wqec, &wqe->generic.wqe_com, 0);
	if (q->phba->sli3_options & LPFC_SLI4_PHWQ_ENABLED)
		bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id);
	lpfc_sli4_pcimem_bcopy(wqe, temp_wqe, q->entry_size);
	if (q->dpp_enable && q->phba->cfg_enable_dpp) {
		/* write to DPP aperture taking advatage of Combined Writes */
		tmp = (uint8_t *)temp_wqe;
#ifdef __raw_writeq
		for (i = 0; i < q->entry_size; i += sizeof(uint64_t))
			__raw_writeq(*((uint64_t *)(tmp + i)),
					q->dpp_regaddr + i);
#else
		for (i = 0; i < q->entry_size; i += sizeof(uint32_t))
			__raw_writel(*((uint32_t *)(tmp + i)),
					q->dpp_regaddr + i);
#endif
	}
	/* ensure WQE bcopy and DPP flushed before doorbell write */
	wmb();

	/* Update the host index before invoking device */
	host_index = q->host_index;

	q->host_index = idx;

	/* Ring Doorbell */
	doorbell.word0 = 0;
	if (q->db_format == LPFC_DB_LIST_FORMAT) {
		if (q->dpp_enable && q->phba->cfg_enable_dpp) {
			bf_set(lpfc_if6_wq_db_list_fm_num_posted, &doorbell, 1);
			bf_set(lpfc_if6_wq_db_list_fm_dpp, &doorbell, 1);
			bf_set(lpfc_if6_wq_db_list_fm_dpp_id, &doorbell,
			    q->dpp_id);
			bf_set(lpfc_if6_wq_db_list_fm_id, &doorbell,
			    q->queue_id);
		} else {
			bf_set(lpfc_wq_db_list_fm_num_posted, &doorbell, 1);
			bf_set(lpfc_wq_db_list_fm_id, &doorbell, q->queue_id);

			/* Leave bits <23:16> clear for if_type 6 dpp */
			if_type = bf_get(lpfc_sli_intf_if_type,
					 &q->phba->sli4_hba.sli_intf);
			if (if_type != LPFC_SLI_INTF_IF_TYPE_6)
				bf_set(lpfc_wq_db_list_fm_index, &doorbell,
				       host_index);
		}
	} else if (q->db_format == LPFC_DB_RING_FORMAT) {
		bf_set(lpfc_wq_db_ring_fm_num_posted, &doorbell, 1);
		bf_set(lpfc_wq_db_ring_fm_id, &doorbell, q->queue_id);
	} else {
		return -EINVAL;
	}
	writel(doorbell.word0, q->db_regaddr);

	return 0;
}

/**
 * lpfc_sli4_wq_release - Updates internal hba index for WQ
 * @q: The Work Queue to operate on.
 * @index: The index to advance the hba index to.
 *
 * This routine will update the HBA index of a queue to reflect consumption of
 * Work Queue Entries by the HBA. When the HBA indicates that it has consumed
 * an entry the host calls this function to update the queue's internal
 * pointers. This routine returns the number of entries that were consumed by
 * the HBA.
 **/
static uint32_t
lpfc_sli4_wq_release(struct lpfc_queue *q, uint32_t index)
{
	uint32_t released = 0;

	/* sanity check on queue memory */
	if (unlikely(!q))
		return 0;

	if (q->hba_index == index)
		return 0;
	do {
		q->hba_index = ((q->hba_index + 1) % q->entry_count);
		released++;
	} while (q->hba_index != index);
	return released;
}

/**
 * lpfc_sli4_mq_put - Put a Mailbox Queue Entry on an Mailbox Queue
 * @q: The Mailbox Queue to operate on.
 * @wqe: The Mailbox Queue Entry to put on the Work queue.
 *
 * This routine will copy the contents of @mqe to the next available entry on
 * the @q. This function will then ring the Work Queue Doorbell to signal the
 * HBA to start processing the Work Queue Entry. This function returns 0 if
 * successful. If no entries are available on @q then this function will return
 * -ENOMEM.
 * The caller is expected to hold the hbalock when calling this routine.
 **/
static uint32_t
lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
{
	struct lpfc_mqe *temp_mqe;
	struct lpfc_register doorbell;

	/* sanity check on queue memory */
	if (unlikely(!q))
		return -ENOMEM;
	temp_mqe = lpfc_sli4_qe(q, q->host_index);

	/* If the host has not yet processed the next entry then we are done */
	if (((q->host_index + 1) % q->entry_count) == q->hba_index)
		return -ENOMEM;
	lpfc_sli4_pcimem_bcopy(mqe, temp_mqe, q->entry_size);
	/* Save off the mailbox pointer for completion */
	q->phba->mbox = (MAILBOX_t *)temp_mqe;

	/* Update the host index before invoking device */
	q->host_index = ((q->host_index + 1) % q->entry_count);

	/* Ring Doorbell */
	doorbell.word0 = 0;
	bf_set(lpfc_mq_doorbell_num_posted, &doorbell, 1);
	bf_set(lpfc_mq_doorbell_id, &doorbell, q->queue_id);
	writel(doorbell.word0, q->phba->sli4_hba.MQDBregaddr);
	return 0;
}

/**
 * lpfc_sli4_mq_release - Updates internal hba index for MQ
 * @q: The Mailbox Queue to operate on.
 *
 * This routine will update the HBA index of a queue to reflect consumption of
 * a Mailbox Queue Entry by the HBA. When the HBA indicates that it has consumed
 * an entry the host calls this function to update the queue's internal
 * pointers. This routine returns the number of entries that were consumed by
 * the HBA.
 **/
static uint32_t
lpfc_sli4_mq_release(struct lpfc_queue *q)
{
	/* sanity check on queue memory */
	if (unlikely(!q))
		return 0;

	/* Clear the mailbox pointer for completion */
	q->phba->mbox = NULL;
	q->hba_index = ((q->hba_index + 1) % q->entry_count);
	return 1;
}

/**
 * lpfc_sli4_eq_get - Gets the next valid EQE from a EQ
 * @q: The Event Queue to get the first valid EQE from
 *
 * This routine will get the first valid Event Queue Entry from @q, update
 * the queue's internal hba index, and return the EQE. If no valid EQEs are in
 * the Queue (no more work to do), or the Queue is full of EQEs that have been
 * processed, but not popped back to the HBA then this routine will return NULL.
 **/
static struct lpfc_eqe *
lpfc_sli4_eq_get(struct lpfc_queue *q)
{
	struct lpfc_eqe *eqe;

	/* sanity check on queue memory */
	if (unlikely(!q))
		return NULL;
	eqe = lpfc_sli4_qe(q, q->host_index);

	/* If the next EQE is not valid then we are done */
	if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid)
		return NULL;

	/*
	 * insert barrier for instruction interlock : data from the hardware
	 * must have the valid bit checked before it can be copied and acted
	 * upon. Speculative instructions were allowing a bcopy at the start
	 * of lpfc_sli4_fp_handle_wcqe(), which is called immediately
	 * after our return, to copy data before the valid bit check above
	 * was done. As such, some of the copied data was stale. The barrier
	 * ensures the check is before any data is copied.
	 */
	mb();
	return eqe;
}

/**
 * lpfc_sli4_eq_clr_intr - Turn off interrupts from this EQ
 * @q: The Event Queue to disable interrupts
 *
 **/
void
lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
{
	struct lpfc_register doorbell;

	doorbell.word0 = 0;
	bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
	bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
		(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
	bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
}

/**
 * lpfc_sli4_if6_eq_clr_intr - Turn off interrupts from this EQ
 * @q: The Event Queue to disable interrupts
 *
 **/
void
lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q)
{
	struct lpfc_register doorbell;

	doorbell.word0 = 0;
	bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
}

/**
 * lpfc_sli4_write_eq_db - write EQ DB for eqe's consumed or arm state
 * @phba: adapter with EQ
 * @q: The Event Queue that the host has completed processing for.
 * @count: Number of elements that have been consumed
 * @arm: Indicates whether the host wants to arms this CQ.
 *
 * This routine will notify the HBA, by ringing the doorbell, that count
 * number of EQEs have been processed. The @arm parameter indicates whether
 * the queue should be rearmed when ringing the doorbell.
 **/
void
lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
		     uint32_t count, bool arm)
{
	struct lpfc_register doorbell;

	/* sanity check on queue memory */
	if (unlikely(!q || (count == 0 && !arm)))
		return;

	/* ring doorbell for number popped */
	doorbell.word0 = 0;
	if (arm) {
		bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
		bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
	}
	bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
	bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
			(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
	bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
	/* PCI read to flush PCI pipeline on re-arming for INTx mode */
	if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
		readl(q->phba->sli4_hba.EQDBregaddr);
}

/**
 * lpfc_sli4_if6_write_eq_db - write EQ DB for eqe's consumed or arm state
 * @phba: adapter with EQ
 * @q: The Event Queue that the host has completed processing for.
 * @count: Number of elements that have been consumed
 * @arm: Indicates whether the host wants to arms this CQ.
 *
 * This routine will notify the HBA, by ringing the doorbell, that count
 * number of EQEs have been processed. The @arm parameter indicates whether
 * the queue should be rearmed when ringing the doorbell.
 **/
void
lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
			  uint32_t count, bool arm)
{
	struct lpfc_register doorbell;

	/* sanity check on queue memory */
	if (unlikely(!q || (count == 0 && !arm)))
		return;

	/* ring doorbell for number popped */
	doorbell.word0 = 0;
	if (arm)
		bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1);
	bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, count);
	bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
	writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
	/* PCI read to flush PCI pipeline on re-arming for INTx mode */
	if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
		readl(q->phba->sli4_hba.EQDBregaddr);
}

static void
__lpfc_sli4_consume_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq,
			struct lpfc_eqe *eqe)
{
	if (!phba->sli4_hba.pc_sli4_params.eqav)
		bf_set_le32(lpfc_eqe_valid, eqe, 0);

	eq->host_index = ((eq->host_index + 1) % eq->entry_count);

	/* if the index wrapped around, toggle the valid bit */
	if (phba->sli4_hba.pc_sli4_params.eqav && !eq->host_index)
		eq->qe_valid = (eq->qe_valid) ? 0 : 1;
}

static void
lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq)
{
	struct lpfc_eqe *eqe;
	uint32_t count = 0;

	/* walk all the EQ entries and drop on the floor */
	eqe = lpfc_sli4_eq_get(eq);
	while (eqe) {
		__lpfc_sli4_consume_eqe(phba, eq, eqe);
		count++;
		eqe = lpfc_sli4_eq_get(eq);
	}

	/* Clear and re-arm the EQ */
	phba->sli4_hba.sli4_write_eq_db(phba, eq, count, LPFC_QUEUE_REARM);
}

static int
lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq)
{
	struct lpfc_eqe *eqe;
	int count = 0, consumed = 0;

	if (cmpxchg(&eq->queue_claimed, 0, 1) != 0)
		goto rearm_and_exit;

	eqe = lpfc_sli4_eq_get(eq);
	while (eqe) {
		lpfc_sli4_hba_handle_eqe(phba, eq, eqe);
		__lpfc_sli4_consume_eqe(phba, eq, eqe);

		consumed++;
		if (!(++count % eq->max_proc_limit))
			break;

		if (!(count % eq->notify_interval)) {
			phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed,
							LPFC_QUEUE_NOARM);
			consumed = 0;
		}

		eqe = lpfc_sli4_eq_get(eq);
	}
	eq->EQ_processed += count;

	/* Track the max number of EQEs processed in 1 intr */
	if (count > eq->EQ_max_eqe)
		eq->EQ_max_eqe = count;

	eq->queue_claimed = 0;

rearm_and_exit:
	/* Always clear and re-arm the EQ */
	phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, LPFC_QUEUE_REARM);

	return count;
}

/**
 * lpfc_sli4_cq_get - Gets the next valid CQE from a CQ
 * @q: The Completion Queue to get the first valid CQE from
 *
 * This routine will get the first valid Completion Queue Entry from @q, update
 * the queue's internal hba index, and return the CQE. If no valid CQEs are in
 * the Queue (no more work to do), or the Queue is full of CQEs that have been
 * processed, but not popped back to the HBA then this routine will return NULL.
 **/
static struct lpfc_cqe *
lpfc_sli4_cq_get(struct lpfc_queue *q)
{
	struct lpfc_cqe *cqe;

	/* sanity check on queue memory */
	if (unlikely(!q))
		return NULL;
	cqe = lpfc_sli4_qe(q, q->host_index);

	/* If the next CQE is not valid then we are done */
	if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid)
		return NULL;

	/*
	 * insert barrier for instruction interlock : data from the hardware
	 * must have the valid bit checked before it can be copied and acted
	 * upon. Given what was seen in lpfc_sli4_cq_get() of speculative
	 * instructions allowing action on content before valid bit checked,
	 * add barrier here as well. May not be needed as "content" is a
	 * single 32-bit entity here (vs multi word structure for cq's).
	 */
	mb();
	return cqe;
}

static void
__lpfc_sli4_consume_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			struct lpfc_cqe *cqe)
{
	if (!phba->sli4_hba.pc_sli4_params.cqav)
		bf_set_le32(lpfc_cqe_valid, cqe, 0);

	cq->host_index = ((cq->host_index + 1) % cq->entry_count);

	/* if the index wrapped around, toggle the valid bit */
	if (phba->sli4_hba.pc_sli4_params.cqav && !cq->host_index)
		cq->qe_valid = (cq->qe_valid) ? 0 : 1;
}

/**
 * lpfc_sli4_write_cq_db - write cq DB for entries consumed or arm state.
 * @phba: the adapter with the CQ
 * @q: The Completion Queue that the host has completed processing for.
 * @count: the number of elements that were consumed
 * @arm: Indicates whether the host wants to arms this CQ.
 *
 * This routine will notify the HBA, by ringing the doorbell, that the
 * CQEs have been processed. The @arm parameter specifies whether the
 * queue should be rearmed when ringing the doorbell.
 **/
void
lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
		     uint32_t count, bool arm)
{
	struct lpfc_register doorbell;

	/* sanity check on queue memory */
	if (unlikely(!q || (count == 0 && !arm)))
		return;

	/* ring doorbell for number popped */
	doorbell.word0 = 0;
	if (arm)
		bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
	bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count);
	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION);
	bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell,
			(q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT));
	bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id);
	writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
}

/**
 * lpfc_sli4_if6_write_cq_db - write cq DB for entries consumed or arm state.
 * @phba: the adapter with the CQ
 * @q: The Completion Queue that the host has completed processing for.
 * @count: the number of elements that were consumed
 * @arm: Indicates whether the host wants to arms this CQ.
 *
 * This routine will notify the HBA, by ringing the doorbell, that the
 * CQEs have been processed. The @arm parameter specifies whether the
 * queue should be rearmed when ringing the doorbell.
 **/
void
lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
			 uint32_t count, bool arm)
{
	struct lpfc_register doorbell;

	/* sanity check on queue memory */
	if (unlikely(!q || (count == 0 && !arm)))
		return;

	/* ring doorbell for number popped */
	doorbell.word0 = 0;
	if (arm)
		bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1);
	bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, count);
	bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id);
	writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
}

/**
 * lpfc_sli4_rq_put - Put a Receive Buffer Queue Entry on a Receive Queue
 * @q: The Header Receive Queue to operate on.
 * @wqe: The Receive Queue Entry to put on the Receive queue.
 *
 * This routine will copy the contents of @wqe to the next available entry on
 * the @q. This function will then ring the Receive Queue Doorbell to signal the
 * HBA to start processing the Receive Queue Entry. This function returns the
 * index that the rqe was copied to if successful. If no entries are available
 * on @q then this function will return -ENOMEM.
 * The caller is expected to hold the hbalock when calling this routine.
 **/
int
lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
		 struct lpfc_rqe *hrqe, struct lpfc_rqe *drqe)
{
	struct lpfc_rqe *temp_hrqe;
	struct lpfc_rqe *temp_drqe;
	struct lpfc_register doorbell;
	int hq_put_index;
	int dq_put_index;

	/* sanity check on queue memory */
	if (unlikely(!hq) || unlikely(!dq))
		return -ENOMEM;
	hq_put_index = hq->host_index;
	dq_put_index = dq->host_index;
	temp_hrqe = lpfc_sli4_qe(hq, hq_put_index);
	temp_drqe = lpfc_sli4_qe(dq, dq_put_index);

	if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ)
		return -EINVAL;
	if (hq_put_index != dq_put_index)
		return -EINVAL;
	/* If the host has not yet processed the next entry then we are done */
	if (((hq_put_index + 1) % hq->entry_count) == hq->hba_index)
		return -EBUSY;
	lpfc_sli4_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
	lpfc_sli4_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);

	/* Update the host index to point to the next slot */
	hq->host_index = ((hq_put_index + 1) % hq->entry_count);
	dq->host_index = ((dq_put_index + 1) % dq->entry_count);
	hq->RQ_buf_posted++;

	/* Ring The Header Receive Queue Doorbell */
	if (!(hq->host_index % hq->notify_interval)) {
		doorbell.word0 = 0;
		if (hq->db_format == LPFC_DB_RING_FORMAT) {
			bf_set(lpfc_rq_db_ring_fm_num_posted, &doorbell,
			       hq->notify_interval);
			bf_set(lpfc_rq_db_ring_fm_id, &doorbell, hq->queue_id);
		} else if (hq->db_format == LPFC_DB_LIST_FORMAT) {
			bf_set(lpfc_rq_db_list_fm_num_posted, &doorbell,
			       hq->notify_interval);
			bf_set(lpfc_rq_db_list_fm_index, &doorbell,
			       hq->host_index);
			bf_set(lpfc_rq_db_list_fm_id, &doorbell, hq->queue_id);
		} else {
			return -EINVAL;
		}
		writel(doorbell.word0, hq->db_regaddr);
	}
	return hq_put_index;
}

/**
 * lpfc_sli4_rq_release - Updates internal hba index for RQ
 * @q: The Header Receive Queue to operate on.
 *
 * This routine will update the HBA index of a queue to reflect consumption of
 * one Receive Queue Entry by the HBA. When the HBA indicates that it has
 * consumed an entry the host calls this function to update the queue's
 * internal pointers. This routine returns the number of entries that were
 * consumed by the HBA.
 **/
static uint32_t
lpfc_sli4_rq_release(struct lpfc_queue *hq, struct lpfc_queue *dq)
{
	/* sanity check on queue memory */
	if (unlikely(!hq) || unlikely(!dq))
		return 0;

	if ((hq->type != LPFC_HRQ) || (dq->type != LPFC_DRQ))
		return 0;
	hq->hba_index = ((hq->hba_index + 1) % hq->entry_count);
	dq->hba_index = ((dq->hba_index + 1) % dq->entry_count);
	return 1;
}

/**
 * lpfc_cmd_iocb - Get next command iocb entry in the ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function returns pointer to next command iocb entry
 * in the command ring. The caller must hold hbalock to prevent
 * other threads consume the next command iocb.
 * SLI-2/SLI-3 provide different sized iocbs.
 **/
static inline IOCB_t *
lpfc_cmd_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	return (IOCB_t *) (((char *) pring->sli.sli3.cmdringaddr) +
			   pring->sli.sli3.cmdidx * phba->iocb_cmd_size);
}

/**
 * lpfc_resp_iocb - Get next response iocb entry in the ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function returns pointer to next response iocb entry
 * in the response ring. The caller must hold hbalock to make sure
 * that no other thread consume the next response iocb.
 * SLI-2/SLI-3 provide different sized iocbs.
 **/
static inline IOCB_t *
lpfc_resp_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	return (IOCB_t *) (((char *) pring->sli.sli3.rspringaddr) +
			   pring->sli.sli3.rspidx * phba->iocb_rsp_size);
}

/**
 * __lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
 * @phba: Pointer to HBA context object.
 *
 * This function is called with hbalock held. This function
 * allocates a new driver iocb object from the iocb pool. If the
 * allocation is successful, it returns pointer to the newly
 * allocated iocb object else it returns NULL.
 **/
struct lpfc_iocbq *
__lpfc_sli_get_iocbq(struct lpfc_hba *phba)
{
	struct list_head *lpfc_iocb_list = &phba->lpfc_iocb_list;
	struct lpfc_iocbq * iocbq = NULL;

	lockdep_assert_held(&phba->hbalock);

	list_remove_head(lpfc_iocb_list, iocbq, struct lpfc_iocbq, list);
	if (iocbq)
		phba->iocb_cnt++;
	if (phba->iocb_cnt > phba->iocb_max)
		phba->iocb_max = phba->iocb_cnt;
	return iocbq;
}

/**
 * __lpfc_clear_active_sglq - Remove the active sglq for this XRI.
 * @phba: Pointer to HBA context object.
 * @xritag: XRI value.
 *
 * This function clears the sglq pointer from the array of acive
 * sglq's. The xritag that is passed in is used to index into the
 * array. Before the xritag can be used it needs to be adjusted
 * by subtracting the xribase.
 *
 * Returns sglq ponter = success, NULL = Failure.
 **/
struct lpfc_sglq *
__lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
{
	struct lpfc_sglq *sglq;

	sglq = phba->sli4_hba.lpfc_sglq_active_list[xritag];
	phba->sli4_hba.lpfc_sglq_active_list[xritag] = NULL;
	return sglq;
}

/**
 * __lpfc_get_active_sglq - Get the active sglq for this XRI.
 * @phba: Pointer to HBA context object.
 * @xritag: XRI value.
 *
 * This function returns the sglq pointer from the array of acive
 * sglq's. The xritag that is passed in is used to index into the
 * array. Before the xritag can be used it needs to be adjusted
 * by subtracting the xribase.
 *
 * Returns sglq ponter = success, NULL = Failure.
 **/
struct lpfc_sglq *
__lpfc_get_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
{
	struct lpfc_sglq *sglq;

	sglq =  phba->sli4_hba.lpfc_sglq_active_list[xritag];
	return sglq;
}

/**
 * lpfc_clr_rrq_active - Clears RRQ active bit in xri_bitmap.
 * @phba: Pointer to HBA context object.
 * @xritag: xri used in this exchange.
 * @rrq: The RRQ to be cleared.
 *
 **/
void
lpfc_clr_rrq_active(struct lpfc_hba *phba,
		    uint16_t xritag,
		    struct lpfc_node_rrq *rrq)
{
	struct lpfc_nodelist *ndlp = NULL;

	if ((rrq->vport) && NLP_CHK_NODE_ACT(rrq->ndlp))
		ndlp = lpfc_findnode_did(rrq->vport, rrq->nlp_DID);

	/* The target DID could have been swapped (cable swap)
	 * we should use the ndlp from the findnode if it is
	 * available.
	 */
	if ((!ndlp) && rrq->ndlp)
		ndlp = rrq->ndlp;

	if (!ndlp)
		goto out;

	if (test_and_clear_bit(xritag, ndlp->active_rrqs_xri_bitmap)) {
		rrq->send_rrq = 0;
		rrq->xritag = 0;
		rrq->rrq_stop_time = 0;
	}
out:
	mempool_free(rrq, phba->rrq_pool);
}

/**
 * lpfc_handle_rrq_active - Checks if RRQ has waithed RATOV.
 * @phba: Pointer to HBA context object.
 *
 * This function is called with hbalock held. This function
 * Checks if stop_time (ratov from setting rrq active) has
 * been reached, if it has and the send_rrq flag is set then
 * it will call lpfc_send_rrq. If the send_rrq flag is not set
 * then it will just call the routine to clear the rrq and
 * free the rrq resource.
 * The timer is set to the next rrq that is going to expire before
 * leaving the routine.
 *
 **/
void
lpfc_handle_rrq_active(struct lpfc_hba *phba)
{
	struct lpfc_node_rrq *rrq;
	struct lpfc_node_rrq *nextrrq;
	unsigned long next_time;
	unsigned long iflags;
	LIST_HEAD(send_rrq);

	spin_lock_irqsave(&phba->hbalock, iflags);
	phba->hba_flag &= ~HBA_RRQ_ACTIVE;
	next_time = jiffies + msecs_to_jiffies(1000 * (phba->fc_ratov + 1));
	list_for_each_entry_safe(rrq, nextrrq,
				 &phba->active_rrq_list, list) {
		if (time_after(jiffies, rrq->rrq_stop_time))
			list_move(&rrq->list, &send_rrq);
		else if (time_before(rrq->rrq_stop_time, next_time))
			next_time = rrq->rrq_stop_time;
	}
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	if ((!list_empty(&phba->active_rrq_list)) &&
	    (!(phba->pport->load_flag & FC_UNLOADING)))
		mod_timer(&phba->rrq_tmr, next_time);
	list_for_each_entry_safe(rrq, nextrrq, &send_rrq, list) {
		list_del(&rrq->list);
		if (!rrq->send_rrq) {
			/* this call will free the rrq */
			lpfc_clr_rrq_active(phba, rrq->xritag, rrq);
		} else if (lpfc_send_rrq(phba, rrq)) {
			/* if we send the rrq then the completion handler
			*  will clear the bit in the xribitmap.
			*/
			lpfc_clr_rrq_active(phba, rrq->xritag,
					    rrq);
		}
	}
}

/**
 * lpfc_get_active_rrq - Get the active RRQ for this exchange.
 * @vport: Pointer to vport context object.
 * @xri: The xri used in the exchange.
 * @did: The targets DID for this exchange.
 *
 * returns NULL = rrq not found in the phba->active_rrq_list.
 *         rrq = rrq for this xri and target.
 **/
struct lpfc_node_rrq *
lpfc_get_active_rrq(struct lpfc_vport *vport, uint16_t xri, uint32_t did)
{
	struct lpfc_hba *phba = vport->phba;
	struct lpfc_node_rrq *rrq;
	struct lpfc_node_rrq *nextrrq;
	unsigned long iflags;

	if (phba->sli_rev != LPFC_SLI_REV4)
		return NULL;
	spin_lock_irqsave(&phba->hbalock, iflags);
	list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) {
		if (rrq->vport == vport && rrq->xritag == xri &&
				rrq->nlp_DID == did){
			list_del(&rrq->list);
			spin_unlock_irqrestore(&phba->hbalock, iflags);
			return rrq;
		}
	}
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	return NULL;
}

/**
 * lpfc_cleanup_vports_rrqs - Remove and clear the active RRQ for this vport.
 * @vport: Pointer to vport context object.
 * @ndlp: Pointer to the lpfc_node_list structure.
 * If ndlp is NULL Remove all active RRQs for this vport from the
 * phba->active_rrq_list and clear the rrq.
 * If ndlp is not NULL then only remove rrqs for this vport & this ndlp.
 **/
void
lpfc_cleanup_vports_rrqs(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)

{
	struct lpfc_hba *phba = vport->phba;
	struct lpfc_node_rrq *rrq;
	struct lpfc_node_rrq *nextrrq;
	unsigned long iflags;
	LIST_HEAD(rrq_list);

	if (phba->sli_rev != LPFC_SLI_REV4)
		return;
	if (!ndlp) {
		lpfc_sli4_vport_delete_els_xri_aborted(vport);
		lpfc_sli4_vport_delete_fcp_xri_aborted(vport);
	}
	spin_lock_irqsave(&phba->hbalock, iflags);
	list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list)
		if ((rrq->vport == vport) && (!ndlp  || rrq->ndlp == ndlp))
			list_move(&rrq->list, &rrq_list);
	spin_unlock_irqrestore(&phba->hbalock, iflags);

	list_for_each_entry_safe(rrq, nextrrq, &rrq_list, list) {
		list_del(&rrq->list);
		lpfc_clr_rrq_active(phba, rrq->xritag, rrq);
	}
}

/**
 * lpfc_test_rrq_active - Test RRQ bit in xri_bitmap.
 * @phba: Pointer to HBA context object.
 * @ndlp: Targets nodelist pointer for this exchange.
 * @xritag the xri in the bitmap to test.
 *
 * This function returns:
 * 0 = rrq not active for this xri
 * 1 = rrq is valid for this xri.
 **/
int
lpfc_test_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
			uint16_t  xritag)
{
	if (!ndlp)
		return 0;
	if (!ndlp->active_rrqs_xri_bitmap)
		return 0;
	if (test_bit(xritag, ndlp->active_rrqs_xri_bitmap))
		return 1;
	else
		return 0;
}

/**
 * lpfc_set_rrq_active - set RRQ active bit in xri_bitmap.
 * @phba: Pointer to HBA context object.
 * @ndlp: nodelist pointer for this target.
 * @xritag: xri used in this exchange.
 * @rxid: Remote Exchange ID.
 * @send_rrq: Flag used to determine if we should send rrq els cmd.
 *
 * This function takes the hbalock.
 * The active bit is always set in the active rrq xri_bitmap even
 * if there is no slot avaiable for the other rrq information.
 *
 * returns 0 rrq actived for this xri
 *         < 0 No memory or invalid ndlp.
 **/
int
lpfc_set_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
		    uint16_t xritag, uint16_t rxid, uint16_t send_rrq)
{
	unsigned long iflags;
	struct lpfc_node_rrq *rrq;
	int empty;

	if (!ndlp)
		return -EINVAL;

	if (!phba->cfg_enable_rrq)
		return -EINVAL;

	spin_lock_irqsave(&phba->hbalock, iflags);
	if (phba->pport->load_flag & FC_UNLOADING) {
		phba->hba_flag &= ~HBA_RRQ_ACTIVE;
		goto out;
	}

	/*
	 * set the active bit even if there is no mem available.
	 */
	if (NLP_CHK_FREE_REQ(ndlp))
		goto out;

	if (ndlp->vport && (ndlp->vport->load_flag & FC_UNLOADING))
		goto out;

	if (!ndlp->active_rrqs_xri_bitmap)
		goto out;

	if (test_and_set_bit(xritag, ndlp->active_rrqs_xri_bitmap))
		goto out;

	spin_unlock_irqrestore(&phba->hbalock, iflags);
	rrq = mempool_alloc(phba->rrq_pool, GFP_KERNEL);
	if (!rrq) {
		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"3155 Unable to allocate RRQ xri:0x%x rxid:0x%x"
				" DID:0x%x Send:%d\n",
				xritag, rxid, ndlp->nlp_DID, send_rrq);
		return -EINVAL;
	}
	if (phba->cfg_enable_rrq == 1)
		rrq->send_rrq = send_rrq;
	else
		rrq->send_rrq = 0;
	rrq->xritag = xritag;
	rrq->rrq_stop_time = jiffies +
				msecs_to_jiffies(1000 * (phba->fc_ratov + 1));
	rrq->ndlp = ndlp;
	rrq->nlp_DID = ndlp->nlp_DID;
	rrq->vport = ndlp->vport;
	rrq->rxid = rxid;
	spin_lock_irqsave(&phba->hbalock, iflags);
	empty = list_empty(&phba->active_rrq_list);
	list_add_tail(&rrq->list, &phba->active_rrq_list);
	phba->hba_flag |= HBA_RRQ_ACTIVE;
	if (empty)
		lpfc_worker_wake_up(phba);
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	return 0;
out:
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"2921 Can't set rrq active xri:0x%x rxid:0x%x"
			" DID:0x%x Send:%d\n",
			xritag, rxid, ndlp->nlp_DID, send_rrq);
	return -EINVAL;
}

/**
 * __lpfc_sli_get_els_sglq - Allocates an iocb object from sgl pool
 * @phba: Pointer to HBA context object.
 * @piocb: Pointer to the iocbq.
 *
 * The driver calls this function with either the nvme ls ring lock
 * or the fc els ring lock held depending on the iocb usage.  This function
 * gets a new driver sglq object from the sglq list. If the list is not empty
 * then it is successful, it returns pointer to the newly allocated sglq
 * object else it returns NULL.
 **/
static struct lpfc_sglq *
__lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
{
	struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list;
	struct lpfc_sglq *sglq = NULL;
	struct lpfc_sglq *start_sglq = NULL;
	struct lpfc_io_buf *lpfc_cmd;
	struct lpfc_nodelist *ndlp;
	struct lpfc_sli_ring *pring = NULL;
	int found = 0;

	if (piocbq->iocb_flag & LPFC_IO_NVME_LS)
		pring =  phba->sli4_hba.nvmels_wq->pring;
	else
		pring = lpfc_phba_elsring(phba);

	lockdep_assert_held(&pring->ring_lock);

	if (piocbq->iocb_flag &  LPFC_IO_FCP) {
		lpfc_cmd = (struct lpfc_io_buf *) piocbq->context1;
		ndlp = lpfc_cmd->rdata->pnode;
	} else  if ((piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) &&
			!(piocbq->iocb_flag & LPFC_IO_LIBDFC)) {
		ndlp = piocbq->context_un.ndlp;
	} else  if (piocbq->iocb_flag & LPFC_IO_LIBDFC) {
		if (piocbq->iocb_flag & LPFC_IO_LOOPBACK)
			ndlp = NULL;
		else
			ndlp = piocbq->context_un.ndlp;
	} else {
		ndlp = piocbq->context1;
	}

	spin_lock(&phba->sli4_hba.sgl_list_lock);
	list_remove_head(lpfc_els_sgl_list, sglq, struct lpfc_sglq, list);
	start_sglq = sglq;
	while (!found) {
		if (!sglq)
			break;
		if (ndlp && ndlp->active_rrqs_xri_bitmap &&
		    test_bit(sglq->sli4_lxritag,
		    ndlp->active_rrqs_xri_bitmap)) {
			/* This xri has an rrq outstanding for this DID.
			 * put it back in the list and get another xri.
			 */
			list_add_tail(&sglq->list, lpfc_els_sgl_list);
			sglq = NULL;
			list_remove_head(lpfc_els_sgl_list, sglq,
						struct lpfc_sglq, list);
			if (sglq == start_sglq) {
				list_add_tail(&sglq->list, lpfc_els_sgl_list);
				sglq = NULL;
				break;
			} else
				continue;
		}
		sglq->ndlp = ndlp;
		found = 1;
		phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
		sglq->state = SGL_ALLOCATED;
	}
	spin_unlock(&phba->sli4_hba.sgl_list_lock);
	return sglq;
}

/**
 * __lpfc_sli_get_nvmet_sglq - Allocates an iocb object from sgl pool
 * @phba: Pointer to HBA context object.
 * @piocb: Pointer to the iocbq.
 *
 * This function is called with the sgl_list lock held. This function
 * gets a new driver sglq object from the sglq list. If the
 * list is not empty then it is successful, it returns pointer to the newly
 * allocated sglq object else it returns NULL.
 **/
struct lpfc_sglq *
__lpfc_sli_get_nvmet_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq)
{
	struct list_head *lpfc_nvmet_sgl_list;
	struct lpfc_sglq *sglq = NULL;

	lpfc_nvmet_sgl_list = &phba->sli4_hba.lpfc_nvmet_sgl_list;

	lockdep_assert_held(&phba->sli4_hba.sgl_list_lock);

	list_remove_head(lpfc_nvmet_sgl_list, sglq, struct lpfc_sglq, list);
	if (!sglq)
		return NULL;
	phba->sli4_hba.lpfc_sglq_active_list[sglq->sli4_lxritag] = sglq;
	sglq->state = SGL_ALLOCATED;
	return sglq;
}

/**
 * lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
 * @phba: Pointer to HBA context object.
 *
 * This function is called with no lock held. This function
 * allocates a new driver iocb object from the iocb pool. If the
 * allocation is successful, it returns pointer to the newly
 * allocated iocb object else it returns NULL.
 **/
struct lpfc_iocbq *
lpfc_sli_get_iocbq(struct lpfc_hba *phba)
{
	struct lpfc_iocbq * iocbq = NULL;
	unsigned long iflags;

	spin_lock_irqsave(&phba->hbalock, iflags);
	iocbq = __lpfc_sli_get_iocbq(phba);
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	return iocbq;
}

/**
 * __lpfc_sli_release_iocbq_s4 - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with hbalock held to release driver
 * iocb object to the iocb pool. The iotag in the iocb object
 * does not change for each use of the iocb object. This function
 * clears all other fields of the iocb object when it is freed.
 * The sqlq structure that holds the xritag and phys and virtual
 * mappings for the scatter gather list is retrieved from the
 * active array of sglq. The get of the sglq pointer also clears
 * the entry in the array. If the status of the IO indiactes that
 * this IO was aborted then the sglq entry it put on the
 * lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the
 * IO has good status or fails for any other reason then the sglq
 * entry is added to the free list (lpfc_els_sgl_list).
 **/
static void
__lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
	struct lpfc_sglq *sglq;
	size_t start_clean = offsetof(struct lpfc_iocbq, iocb);
	unsigned long iflag = 0;
	struct lpfc_sli_ring *pring;

	lockdep_assert_held(&phba->hbalock);

	if (iocbq->sli4_xritag == NO_XRI)
		sglq = NULL;
	else
		sglq = __lpfc_clear_active_sglq(phba, iocbq->sli4_lxritag);


	if (sglq)  {
		if (iocbq->iocb_flag & LPFC_IO_NVMET) {
			spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
					  iflag);
			sglq->state = SGL_FREED;
			sglq->ndlp = NULL;
			list_add_tail(&sglq->list,
				      &phba->sli4_hba.lpfc_nvmet_sgl_list);
			spin_unlock_irqrestore(
				&phba->sli4_hba.sgl_list_lock, iflag);
			goto out;
		}

		pring = phba->sli4_hba.els_wq->pring;
		if ((iocbq->iocb_flag & LPFC_EXCHANGE_BUSY) &&
			(sglq->state != SGL_XRI_ABORTED)) {
			spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
					  iflag);
			list_add(&sglq->list,
				 &phba->sli4_hba.lpfc_abts_els_sgl_list);
			spin_unlock_irqrestore(
				&phba->sli4_hba.sgl_list_lock, iflag);
		} else {
			spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock,
					  iflag);
			sglq->state = SGL_FREED;
			sglq->ndlp = NULL;
			list_add_tail(&sglq->list,
				      &phba->sli4_hba.lpfc_els_sgl_list);
			spin_unlock_irqrestore(
				&phba->sli4_hba.sgl_list_lock, iflag);

			/* Check if TXQ queue needs to be serviced */
			if (!list_empty(&pring->txq))
				lpfc_worker_wake_up(phba);
		}
	}

out:
	/*
	 * Clean all volatile data fields, preserve iotag and node struct.
	 */
	memset((char *)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean);
	iocbq->sli4_lxritag = NO_XRI;
	iocbq->sli4_xritag = NO_XRI;
	iocbq->iocb_flag &= ~(LPFC_IO_NVME | LPFC_IO_NVMET |
			      LPFC_IO_NVME_LS);
	list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
}


/**
 * __lpfc_sli_release_iocbq_s3 - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with hbalock held to release driver
 * iocb object to the iocb pool. The iotag in the iocb object
 * does not change for each use of the iocb object. This function
 * clears all other fields of the iocb object when it is freed.
 **/
static void
__lpfc_sli_release_iocbq_s3(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
	size_t start_clean = offsetof(struct lpfc_iocbq, iocb);

	lockdep_assert_held(&phba->hbalock);

	/*
	 * Clean all volatile data fields, preserve iotag and node struct.
	 */
	memset((char*)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean);
	iocbq->sli4_xritag = NO_XRI;
	list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
}

/**
 * __lpfc_sli_release_iocbq - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with hbalock held to release driver
 * iocb object to the iocb pool. The iotag in the iocb object
 * does not change for each use of the iocb object. This function
 * clears all other fields of the iocb object when it is freed.
 **/
static void
__lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
	lockdep_assert_held(&phba->hbalock);

	phba->__lpfc_sli_release_iocbq(phba, iocbq);
	phba->iocb_cnt--;
}

/**
 * lpfc_sli_release_iocbq - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with no lock held to release the iocb to
 * iocb pool.
 **/
void
lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
	unsigned long iflags;

	/*
	 * Clean all volatile data fields, preserve iotag and node struct.
	 */
	spin_lock_irqsave(&phba->hbalock, iflags);
	__lpfc_sli_release_iocbq(phba, iocbq);
	spin_unlock_irqrestore(&phba->hbalock, iflags);
}

/**
 * lpfc_sli_cancel_iocbs - Cancel all iocbs from a list.
 * @phba: Pointer to HBA context object.
 * @iocblist: List of IOCBs.
 * @ulpstatus: ULP status in IOCB command field.
 * @ulpWord4: ULP word-4 in IOCB command field.
 *
 * This function is called with a list of IOCBs to cancel. It cancels the IOCB
 * on the list by invoking the complete callback function associated with the
 * IOCB with the provided @ulpstatus and @ulpword4 set to the IOCB commond
 * fields.
 **/
void
lpfc_sli_cancel_iocbs(struct lpfc_hba *phba, struct list_head *iocblist,
		      uint32_t ulpstatus, uint32_t ulpWord4)
{
	struct lpfc_iocbq *piocb;

	while (!list_empty(iocblist)) {
		list_remove_head(iocblist, piocb, struct lpfc_iocbq, list);
		if (!piocb->iocb_cmpl)
			lpfc_sli_release_iocbq(phba, piocb);
		else {
			piocb->iocb.ulpStatus = ulpstatus;
			piocb->iocb.un.ulpWord[4] = ulpWord4;
			(piocb->iocb_cmpl) (phba, piocb, piocb);
		}
	}
	return;
}

/**
 * lpfc_sli_iocb_cmd_type - Get the iocb type
 * @iocb_cmnd: iocb command code.
 *
 * This function is called by ring event handler function to get the iocb type.
 * This function translates the iocb command to an iocb command type used to
 * decide the final disposition of each completed IOCB.
 * The function returns
 * LPFC_UNKNOWN_IOCB if it is an unsupported iocb
 * LPFC_SOL_IOCB     if it is a solicited iocb completion
 * LPFC_ABORT_IOCB   if it is an abort iocb
 * LPFC_UNSOL_IOCB   if it is an unsolicited iocb
 *
 * The caller is not required to hold any lock.
 **/
static lpfc_iocb_type
lpfc_sli_iocb_cmd_type(uint8_t iocb_cmnd)
{
	lpfc_iocb_type type = LPFC_UNKNOWN_IOCB;

	if (iocb_cmnd > CMD_MAX_IOCB_CMD)
		return 0;

	switch (iocb_cmnd) {
	case CMD_XMIT_SEQUENCE_CR:
	case CMD_XMIT_SEQUENCE_CX:
	case CMD_XMIT_BCAST_CN:
	case CMD_XMIT_BCAST_CX:
	case CMD_ELS_REQUEST_CR:
	case CMD_ELS_REQUEST_CX:
	case CMD_CREATE_XRI_CR:
	case CMD_CREATE_XRI_CX:
	case CMD_GET_RPI_CN:
	case CMD_XMIT_ELS_RSP_CX:
	case CMD_GET_RPI_CR:
	case CMD_FCP_IWRITE_CR:
	case CMD_FCP_IWRITE_CX:
	case CMD_FCP_IREAD_CR:
	case CMD_FCP_IREAD_CX:
	case CMD_FCP_ICMND_CR:
	case CMD_FCP_ICMND_CX:
	case CMD_FCP_TSEND_CX:
	case CMD_FCP_TRSP_CX:
	case CMD_FCP_TRECEIVE_CX:
	case CMD_FCP_AUTO_TRSP_CX:
	case CMD_ADAPTER_MSG:
	case CMD_ADAPTER_DUMP:
	case CMD_XMIT_SEQUENCE64_CR:
	case CMD_XMIT_SEQUENCE64_CX:
	case CMD_XMIT_BCAST64_CN:
	case CMD_XMIT_BCAST64_CX:
	case CMD_ELS_REQUEST64_CR:
	case CMD_ELS_REQUEST64_CX:
	case CMD_FCP_IWRITE64_CR:
	case CMD_FCP_IWRITE64_CX:
	case CMD_FCP_IREAD64_CR:
	case CMD_FCP_IREAD64_CX:
	case CMD_FCP_ICMND64_CR:
	case CMD_FCP_ICMND64_CX:
	case CMD_FCP_TSEND64_CX:
	case CMD_FCP_TRSP64_CX:
	case CMD_FCP_TRECEIVE64_CX:
	case CMD_GEN_REQUEST64_CR:
	case CMD_GEN_REQUEST64_CX:
	case CMD_XMIT_ELS_RSP64_CX:
	case DSSCMD_IWRITE64_CR:
	case DSSCMD_IWRITE64_CX:
	case DSSCMD_IREAD64_CR:
	case DSSCMD_IREAD64_CX:
		type = LPFC_SOL_IOCB;
		break;
	case CMD_ABORT_XRI_CN:
	case CMD_ABORT_XRI_CX:
	case CMD_CLOSE_XRI_CN:
	case CMD_CLOSE_XRI_CX:
	case CMD_XRI_ABORTED_CX:
	case CMD_ABORT_MXRI64_CN:
	case CMD_XMIT_BLS_RSP64_CX:
		type = LPFC_ABORT_IOCB;
		break;
	case CMD_RCV_SEQUENCE_CX:
	case CMD_RCV_ELS_REQ_CX:
	case CMD_RCV_SEQUENCE64_CX:
	case CMD_RCV_ELS_REQ64_CX:
	case CMD_ASYNC_STATUS:
	case CMD_IOCB_RCV_SEQ64_CX:
	case CMD_IOCB_RCV_ELS64_CX:
	case CMD_IOCB_RCV_CONT64_CX:
	case CMD_IOCB_RET_XRI64_CX:
		type = LPFC_UNSOL_IOCB;
		break;
	case CMD_IOCB_XMIT_MSEQ64_CR:
	case CMD_IOCB_XMIT_MSEQ64_CX:
	case CMD_IOCB_RCV_SEQ_LIST64_CX:
	case CMD_IOCB_RCV_ELS_LIST64_CX:
	case CMD_IOCB_CLOSE_EXTENDED_CN:
	case CMD_IOCB_ABORT_EXTENDED_CN:
	case CMD_IOCB_RET_HBQE64_CN:
	case CMD_IOCB_FCP_IBIDIR64_CR:
	case CMD_IOCB_FCP_IBIDIR64_CX:
	case CMD_IOCB_FCP_ITASKMGT64_CX:
	case CMD_IOCB_LOGENTRY_CN:
	case CMD_IOCB_LOGENTRY_ASYNC_CN:
		printk("%s - Unhandled SLI-3 Command x%x\n",
				__func__, iocb_cmnd);
		type = LPFC_UNKNOWN_IOCB;
		break;
	default:
		type = LPFC_UNKNOWN_IOCB;
		break;
	}

	return type;
}

/**
 * lpfc_sli_ring_map - Issue config_ring mbox for all rings
 * @phba: Pointer to HBA context object.
 *
 * This function is called from SLI initialization code
 * to configure every ring of the HBA's SLI interface. The
 * caller is not required to hold any lock. This function issues
 * a config_ring mailbox command for each ring.
 * This function returns zero if successful else returns a negative
 * error code.
 **/
static int
lpfc_sli_ring_map(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;
	LPFC_MBOXQ_t *pmb;
	MAILBOX_t *pmbox;
	int i, rc, ret = 0;

	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!pmb)
		return -ENOMEM;
	pmbox = &pmb->u.mb;
	phba->link_state = LPFC_INIT_MBX_CMDS;
	for (i = 0; i < psli->num_rings; i++) {
		lpfc_config_ring(phba, i, pmb);
		rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
		if (rc != MBX_SUCCESS) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"0446 Adapter failed to init (%d), "
					"mbxCmd x%x CFG_RING, mbxStatus x%x, "
					"ring %d\n",
					rc, pmbox->mbxCommand,
					pmbox->mbxStatus, i);
			phba->link_state = LPFC_HBA_ERROR;
			ret = -ENXIO;
			break;
		}
	}
	mempool_free(pmb, phba->mbox_mem_pool);
	return ret;
}

/**
 * lpfc_sli_ringtxcmpl_put - Adds new iocb to the txcmplq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @piocb: Pointer to the driver iocb object.
 *
 * The driver calls this function with the hbalock held for SLI3 ports or
 * the ring lock held for SLI4 ports. The function adds the
 * new iocb to txcmplq of the given ring. This function always returns
 * 0. If this function is called for ELS ring, this function checks if
 * there is a vport associated with the ELS command. This function also
 * starts els_tmofunc timer if this is an ELS command.
 **/
static int
lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			struct lpfc_iocbq *piocb)
{
	if (phba->sli_rev == LPFC_SLI_REV4)
		lockdep_assert_held(&pring->ring_lock);
	else
		lockdep_assert_held(&phba->hbalock);

	BUG_ON(!piocb);

	list_add_tail(&piocb->list, &pring->txcmplq);
	piocb->iocb_flag |= LPFC_IO_ON_TXCMPLQ;
	pring->txcmplq_cnt++;

	if ((unlikely(pring->ringno == LPFC_ELS_RING)) &&
	   (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) &&
	   (piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) {
		BUG_ON(!piocb->vport);
		if (!(piocb->vport->load_flag & FC_UNLOADING))
			mod_timer(&piocb->vport->els_tmofunc,
				  jiffies +
				  msecs_to_jiffies(1000 * (phba->fc_ratov << 1)));
	}

	return 0;
}

/**
 * lpfc_sli_ringtx_get - Get first element of the txq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called with hbalock held to get next
 * iocb in txq of the given ring. If there is any iocb in
 * the txq, the function returns first iocb in the list after
 * removing the iocb from the list, else it returns NULL.
 **/
struct lpfc_iocbq *
lpfc_sli_ringtx_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	struct lpfc_iocbq *cmd_iocb;

	lockdep_assert_held(&phba->hbalock);

	list_remove_head((&pring->txq), cmd_iocb, struct lpfc_iocbq, list);
	return cmd_iocb;
}

/**
 * lpfc_sli_next_iocb_slot - Get next iocb slot in the ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called with hbalock held and the caller must post the
 * iocb without releasing the lock. If the caller releases the lock,
 * iocb slot returned by the function is not guaranteed to be available.
 * The function returns pointer to the next available iocb slot if there
 * is available slot in the ring, else it returns NULL.
 * If the get index of the ring is ahead of the put index, the function
 * will post an error attention event to the worker thread to take the
 * HBA to offline state.
 **/
static IOCB_t *
lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
	uint32_t  max_cmd_idx = pring->sli.sli3.numCiocb;

	lockdep_assert_held(&phba->hbalock);

	if ((pring->sli.sli3.next_cmdidx == pring->sli.sli3.cmdidx) &&
	   (++pring->sli.sli3.next_cmdidx >= max_cmd_idx))
		pring->sli.sli3.next_cmdidx = 0;

	if (unlikely(pring->sli.sli3.local_getidx ==
		pring->sli.sli3.next_cmdidx)) {

		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);

		if (unlikely(pring->sli.sli3.local_getidx >= max_cmd_idx)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"0315 Ring %d issue: portCmdGet %d "
					"is bigger than cmd ring %d\n",
					pring->ringno,
					pring->sli.sli3.local_getidx,
					max_cmd_idx);

			phba->link_state = LPFC_HBA_ERROR;
			/*
			 * All error attention handlers are posted to
			 * worker thread
			 */
			phba->work_ha |= HA_ERATT;
			phba->work_hs = HS_FFER3;

			lpfc_worker_wake_up(phba);

			return NULL;
		}

		if (pring->sli.sli3.local_getidx == pring->sli.sli3.next_cmdidx)
			return NULL;
	}

	return lpfc_cmd_iocb(phba, pring);
}

/**
 * lpfc_sli_next_iotag - Get an iotag for the iocb
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function gets an iotag for the iocb. If there is no unused iotag and
 * the iocbq_lookup_len < 0xffff, this function allocates a bigger iotag_lookup
 * array and assigns a new iotag.
 * The function returns the allocated iotag if successful, else returns zero.
 * Zero is not a valid iotag.
 * The caller is not required to hold any lock.
 **/
uint16_t
lpfc_sli_next_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
	struct lpfc_iocbq **new_arr;
	struct lpfc_iocbq **old_arr;
	size_t new_len;
	struct lpfc_sli *psli = &phba->sli;
	uint16_t iotag;

	spin_lock_irq(&phba->hbalock);
	iotag = psli->last_iotag;
	if(++iotag < psli->iocbq_lookup_len) {
		psli->last_iotag = iotag;
		psli->iocbq_lookup[iotag] = iocbq;
		spin_unlock_irq(&phba->hbalock);
		iocbq->iotag = iotag;
		return iotag;
	} else if (psli->iocbq_lookup_len < (0xffff
					   - LPFC_IOCBQ_LOOKUP_INCREMENT)) {
		new_len = psli->iocbq_lookup_len + LPFC_IOCBQ_LOOKUP_INCREMENT;
		spin_unlock_irq(&phba->hbalock);
		new_arr = kcalloc(new_len, sizeof(struct lpfc_iocbq *),
				  GFP_KERNEL);
		if (new_arr) {
			spin_lock_irq(&phba->hbalock);
			old_arr = psli->iocbq_lookup;
			if (new_len <= psli->iocbq_lookup_len) {
				/* highly unprobable case */
				kfree(new_arr);
				iotag = psli->last_iotag;
				if(++iotag < psli->iocbq_lookup_len) {
					psli->last_iotag = iotag;
					psli->iocbq_lookup[iotag] = iocbq;
					spin_unlock_irq(&phba->hbalock);
					iocbq->iotag = iotag;
					return iotag;
				}
				spin_unlock_irq(&phba->hbalock);
				return 0;
			}
			if (psli->iocbq_lookup)
				memcpy(new_arr, old_arr,
				       ((psli->last_iotag  + 1) *
					sizeof (struct lpfc_iocbq *)));
			psli->iocbq_lookup = new_arr;
			psli->iocbq_lookup_len = new_len;
			psli->last_iotag = iotag;
			psli->iocbq_lookup[iotag] = iocbq;
			spin_unlock_irq(&phba->hbalock);
			iocbq->iotag = iotag;
			kfree(old_arr);
			return iotag;
		}
	} else
		spin_unlock_irq(&phba->hbalock);

	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
			"0318 Failed to allocate IOTAG.last IOTAG is %d\n",
			psli->last_iotag);

	return 0;
}

/**
 * lpfc_sli_submit_iocb - Submit an iocb to the firmware
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @iocb: Pointer to iocb slot in the ring.
 * @nextiocb: Pointer to driver iocb object which need to be
 *            posted to firmware.
 *
 * This function is called with hbalock held to post a new iocb to
 * the firmware. This function copies the new iocb to ring iocb slot and
 * updates the ring pointers. It adds the new iocb to txcmplq if there is
 * a completion call back for this iocb else the function will free the
 * iocb object.
 **/
static void
lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
		IOCB_t *iocb, struct lpfc_iocbq *nextiocb)
{
	lockdep_assert_held(&phba->hbalock);
	/*
	 * Set up an iotag
	 */
	nextiocb->iocb.ulpIoTag = (nextiocb->iocb_cmpl) ? nextiocb->iotag : 0;


	if (pring->ringno == LPFC_ELS_RING) {
		lpfc_debugfs_slow_ring_trc(phba,
			"IOCB cmd ring:   wd4:x%08x wd6:x%08x wd7:x%08x",
			*(((uint32_t *) &nextiocb->iocb) + 4),
			*(((uint32_t *) &nextiocb->iocb) + 6),
			*(((uint32_t *) &nextiocb->iocb) + 7));
	}

	/*
	 * Issue iocb command to adapter
	 */
	lpfc_sli_pcimem_bcopy(&nextiocb->iocb, iocb, phba->iocb_cmd_size);
	wmb();
	pring->stats.iocb_cmd++;

	/*
	 * If there is no completion routine to call, we can release the
	 * IOCB buffer back right now. For IOCBs, like QUE_RING_BUF,
	 * that have no rsp ring completion, iocb_cmpl MUST be NULL.
	 */
	if (nextiocb->iocb_cmpl)
		lpfc_sli_ringtxcmpl_put(phba, pring, nextiocb);
	else
		__lpfc_sli_release_iocbq(phba, nextiocb);

	/*
	 * Let the HBA know what IOCB slot will be the next one the
	 * driver will put a command into.
	 */
	pring->sli.sli3.cmdidx = pring->sli.sli3.next_cmdidx;
	writel(pring->sli.sli3.cmdidx, &phba->host_gp[pring->ringno].cmdPutInx);
}

/**
 * lpfc_sli_update_full_ring - Update the chip attention register
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * The caller is not required to hold any lock for calling this function.
 * This function updates the chip attention bits for the ring to inform firmware
 * that there are pending work to be done for this ring and requests an
 * interrupt when there is space available in the ring. This function is
 * called when the driver is unable to post more iocbs to the ring due
 * to unavailability of space in the ring.
 **/
static void
lpfc_sli_update_full_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	int ringno = pring->ringno;

	pring->flag |= LPFC_CALL_RING_AVAILABLE;

	wmb();

	/*
	 * Set ring 'ringno' to SET R0CE_REQ in Chip Att register.
	 * The HBA will tell us when an IOCB entry is available.
	 */
	writel((CA_R0ATT|CA_R0CE_REQ) << (ringno*4), phba->CAregaddr);
	readl(phba->CAregaddr); /* flush */

	pring->stats.iocb_cmd_full++;
}

/**
 * lpfc_sli_update_ring - Update chip attention register
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function updates the chip attention register bit for the
 * given ring to inform HBA that there is more work to be done
 * in this ring. The caller is not required to hold any lock.
 **/
static void
lpfc_sli_update_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	int ringno = pring->ringno;

	/*
	 * Tell the HBA that there is work to do in this ring.
	 */
	if (!(phba->sli3_options & LPFC_SLI3_CRP_ENABLED)) {
		wmb();
		writel(CA_R0ATT << (ringno * 4), phba->CAregaddr);
		readl(phba->CAregaddr); /* flush */
	}
}

/**
 * lpfc_sli_resume_iocb - Process iocbs in the txq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called with hbalock held to post pending iocbs
 * in the txq to the firmware. This function is called when driver
 * detects space available in the ring.
 **/
static void
lpfc_sli_resume_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	IOCB_t *iocb;
	struct lpfc_iocbq *nextiocb;

	lockdep_assert_held(&phba->hbalock);

	/*
	 * Check to see if:
	 *  (a) there is anything on the txq to send
	 *  (b) link is up
	 *  (c) link attention events can be processed (fcp ring only)
	 *  (d) IOCB processing is not blocked by the outstanding mbox command.
	 */

	if (lpfc_is_link_up(phba) &&
	    (!list_empty(&pring->txq)) &&
	    (pring->ringno != LPFC_FCP_RING ||
	     phba->sli.sli_flag & LPFC_PROCESS_LA)) {

		while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
		       (nextiocb = lpfc_sli_ringtx_get(phba, pring)))
			lpfc_sli_submit_iocb(phba, pring, iocb, nextiocb);

		if (iocb)
			lpfc_sli_update_ring(phba, pring);
		else
			lpfc_sli_update_full_ring(phba, pring);
	}

	return;
}

/**
 * lpfc_sli_next_hbq_slot - Get next hbq entry for the HBQ
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 *
 * This function is called with hbalock held to get the next
 * available slot for the given HBQ. If there is free slot
 * available for the HBQ it will return pointer to the next available
 * HBQ entry else it will return NULL.
 **/
static struct lpfc_hbq_entry *
lpfc_sli_next_hbq_slot(struct lpfc_hba *phba, uint32_t hbqno)
{
	struct hbq_s *hbqp = &phba->hbqs[hbqno];

	lockdep_assert_held(&phba->hbalock);

	if (hbqp->next_hbqPutIdx == hbqp->hbqPutIdx &&
	    ++hbqp->next_hbqPutIdx >= hbqp->entry_count)
		hbqp->next_hbqPutIdx = 0;

	if (unlikely(hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)) {
		uint32_t raw_index = phba->hbq_get[hbqno];
		uint32_t getidx = le32_to_cpu(raw_index);

		hbqp->local_hbqGetIdx = getidx;

		if (unlikely(hbqp->local_hbqGetIdx >= hbqp->entry_count)) {
			lpfc_printf_log(phba, KERN_ERR,
					LOG_SLI | LOG_VPORT,
					"1802 HBQ %d: local_hbqGetIdx "
					"%u is > than hbqp->entry_count %u\n",
					hbqno, hbqp->local_hbqGetIdx,
					hbqp->entry_count);

			phba->link_state = LPFC_HBA_ERROR;
			return NULL;
		}

		if (hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)
			return NULL;
	}

	return (struct lpfc_hbq_entry *) phba->hbqs[hbqno].hbq_virt +
			hbqp->hbqPutIdx;
}

/**
 * lpfc_sli_hbqbuf_free_all - Free all the hbq buffers
 * @phba: Pointer to HBA context object.
 *
 * This function is called with no lock held to free all the
 * hbq buffers while uninitializing the SLI interface. It also
 * frees the HBQ buffers returned by the firmware but not yet
 * processed by the upper layers.
 **/
void
lpfc_sli_hbqbuf_free_all(struct lpfc_hba *phba)
{
	struct lpfc_dmabuf *dmabuf, *next_dmabuf;
	struct hbq_dmabuf *hbq_buf;
	unsigned long flags;
	int i, hbq_count;

	hbq_count = lpfc_sli_hbq_count();
	/* Return all memory used by all HBQs */
	spin_lock_irqsave(&phba->hbalock, flags);
	for (i = 0; i < hbq_count; ++i) {
		list_for_each_entry_safe(dmabuf, next_dmabuf,
				&phba->hbqs[i].hbq_buffer_list, list) {
			hbq_buf = container_of(dmabuf, struct hbq_dmabuf, dbuf);
			list_del(&hbq_buf->dbuf.list);
			(phba->hbqs[i].hbq_free_buffer)(phba, hbq_buf);
		}
		phba->hbqs[i].buffer_count = 0;
	}

	/* Mark the HBQs not in use */
	phba->hbq_in_use = 0;
	spin_unlock_irqrestore(&phba->hbalock, flags);
}

/**
 * lpfc_sli_hbq_to_firmware - Post the hbq buffer to firmware
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @hbq_buf: Pointer to HBQ buffer.
 *
 * This function is called with the hbalock held to post a
 * hbq buffer to the firmware. If the function finds an empty
 * slot in the HBQ, it will post the buffer. The function will return
 * pointer to the hbq entry if it successfully post the buffer
 * else it will return NULL.
 **/
static int
lpfc_sli_hbq_to_firmware(struct lpfc_hba *phba, uint32_t hbqno,
			 struct hbq_dmabuf *hbq_buf)
{
	lockdep_assert_held(&phba->hbalock);
	return phba->lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buf);
}

/**
 * lpfc_sli_hbq_to_firmware_s3 - Post the hbq buffer to SLI3 firmware
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @hbq_buf: Pointer to HBQ buffer.
 *
 * This function is called with the hbalock held to post a hbq buffer to the
 * firmware. If the function finds an empty slot in the HBQ, it will post the
 * buffer and place it on the hbq_buffer_list. The function will return zero if
 * it successfully post the buffer else it will return an error.
 **/
static int
lpfc_sli_hbq_to_firmware_s3(struct lpfc_hba *phba, uint32_t hbqno,
			    struct hbq_dmabuf *hbq_buf)
{
	struct lpfc_hbq_entry *hbqe;
	dma_addr_t physaddr = hbq_buf->dbuf.phys;

	lockdep_assert_held(&phba->hbalock);
	/* Get next HBQ entry slot to use */
	hbqe = lpfc_sli_next_hbq_slot(phba, hbqno);
	if (hbqe) {
		struct hbq_s *hbqp = &phba->hbqs[hbqno];

		hbqe->bde.addrHigh = le32_to_cpu(putPaddrHigh(physaddr));
		hbqe->bde.addrLow  = le32_to_cpu(putPaddrLow(physaddr));
		hbqe->bde.tus.f.bdeSize = hbq_buf->total_size;
		hbqe->bde.tus.f.bdeFlags = 0;
		hbqe->bde.tus.w = le32_to_cpu(hbqe->bde.tus.w);
		hbqe->buffer_tag = le32_to_cpu(hbq_buf->tag);
				/* Sync SLIM */
		hbqp->hbqPutIdx = hbqp->next_hbqPutIdx;
		writel(hbqp->hbqPutIdx, phba->hbq_put + hbqno);
				/* flush */
		readl(phba->hbq_put + hbqno);
		list_add_tail(&hbq_buf->dbuf.list, &hbqp->hbq_buffer_list);
		return 0;
	} else
		return -ENOMEM;
}

/**
 * lpfc_sli_hbq_to_firmware_s4 - Post the hbq buffer to SLI4 firmware
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @hbq_buf: Pointer to HBQ buffer.
 *
 * This function is called with the hbalock held to post an RQE to the SLI4
 * firmware. If able to post the RQE to the RQ it will queue the hbq entry to
 * the hbq_buffer_list and return zero, otherwise it will return an error.
 **/
static int
lpfc_sli_hbq_to_firmware_s4(struct lpfc_hba *phba, uint32_t hbqno,
			    struct hbq_dmabuf *hbq_buf)
{
	int rc;
	struct lpfc_rqe hrqe;
	struct lpfc_rqe drqe;
	struct lpfc_queue *hrq;
	struct lpfc_queue *drq;

	if (hbqno != LPFC_ELS_HBQ)
		return 1;
	hrq = phba->sli4_hba.hdr_rq;
	drq = phba->sli4_hba.dat_rq;

	lockdep_assert_held(&phba->hbalock);
	hrqe.address_lo = putPaddrLow(hbq_buf->hbuf.phys);
	hrqe.address_hi = putPaddrHigh(hbq_buf->hbuf.phys);
	drqe.address_lo = putPaddrLow(hbq_buf->dbuf.phys);
	drqe.address_hi = putPaddrHigh(hbq_buf->dbuf.phys);
	rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
	if (rc < 0)
		return rc;
	hbq_buf->tag = (rc | (hbqno << 16));
	list_add_tail(&hbq_buf->dbuf.list, &phba->hbqs[hbqno].hbq_buffer_list);
	return 0;
}

/* HBQ for ELS and CT traffic. */
static struct lpfc_hbq_init lpfc_els_hbq = {
	.rn = 1,
	.entry_count = 256,
	.mask_count = 0,
	.profile = 0,
	.ring_mask = (1 << LPFC_ELS_RING),
	.buffer_count = 0,
	.init_count = 40,
	.add_count = 40,
};

/* Array of HBQs */
struct lpfc_hbq_init *lpfc_hbq_defs[] = {
	&lpfc_els_hbq,
};

/**
 * lpfc_sli_hbqbuf_fill_hbqs - Post more hbq buffers to HBQ
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @count: Number of HBQ buffers to be posted.
 *
 * This function is called with no lock held to post more hbq buffers to the
 * given HBQ. The function returns the number of HBQ buffers successfully
 * posted.
 **/
static int
lpfc_sli_hbqbuf_fill_hbqs(struct lpfc_hba *phba, uint32_t hbqno, uint32_t count)
{
	uint32_t i, posted = 0;
	unsigned long flags;
	struct hbq_dmabuf *hbq_buffer;
	LIST_HEAD(hbq_buf_list);
	if (!phba->hbqs[hbqno].hbq_alloc_buffer)
		return 0;

	if ((phba->hbqs[hbqno].buffer_count + count) >
	    lpfc_hbq_defs[hbqno]->entry_count)
		count = lpfc_hbq_defs[hbqno]->entry_count -
					phba->hbqs[hbqno].buffer_count;
	if (!count)
		return 0;
	/* Allocate HBQ entries */
	for (i = 0; i < count; i++) {
		hbq_buffer = (phba->hbqs[hbqno].hbq_alloc_buffer)(phba);
		if (!hbq_buffer)
			break;
		list_add_tail(&hbq_buffer->dbuf.list, &hbq_buf_list);
	}
	/* Check whether HBQ is still in use */
	spin_lock_irqsave(&phba->hbalock, flags);
	if (!phba->hbq_in_use)
		goto err;
	while (!list_empty(&hbq_buf_list)) {
		list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
				 dbuf.list);
		hbq_buffer->tag = (phba->hbqs[hbqno].buffer_count |
				      (hbqno << 16));
		if (!lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buffer)) {
			phba->hbqs[hbqno].buffer_count++;
			posted++;
		} else
			(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
	}
	spin_unlock_irqrestore(&phba->hbalock, flags);
	return posted;
err:
	spin_unlock_irqrestore(&phba->hbalock, flags);
	while (!list_empty(&hbq_buf_list)) {
		list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
				 dbuf.list);
		(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
	}
	return 0;
}

/**
 * lpfc_sli_hbqbuf_add_hbqs - Post more HBQ buffers to firmware
 * @phba: Pointer to HBA context object.
 * @qno: HBQ number.
 *
 * This function posts more buffers to the HBQ. This function
 * is called with no lock held. The function returns the number of HBQ entries
 * successfully allocated.
 **/
int
lpfc_sli_hbqbuf_add_hbqs(struct lpfc_hba *phba, uint32_t qno)
{
	if (phba->sli_rev == LPFC_SLI_REV4)
		return 0;
	else
		return lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
					 lpfc_hbq_defs[qno]->add_count);
}

/**
 * lpfc_sli_hbqbuf_init_hbqs - Post initial buffers to the HBQ
 * @phba: Pointer to HBA context object.
 * @qno:  HBQ queue number.
 *
 * This function is called from SLI initialization code path with
 * no lock held to post initial HBQ buffers to firmware. The
 * function returns the number of HBQ entries successfully allocated.
 **/
static int
lpfc_sli_hbqbuf_init_hbqs(struct lpfc_hba *phba, uint32_t qno)
{
	if (phba->sli_rev == LPFC_SLI_REV4)
		return lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
					lpfc_hbq_defs[qno]->entry_count);
	else
		return lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
					 lpfc_hbq_defs[qno]->init_count);
}

/**
 * lpfc_sli_hbqbuf_get - Remove the first hbq off of an hbq list
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 *
 * This function removes the first hbq buffer on an hbq list and returns a
 * pointer to that buffer. If it finds no buffers on the list it returns NULL.
 **/
static struct hbq_dmabuf *
lpfc_sli_hbqbuf_get(struct list_head *rb_list)
{
	struct lpfc_dmabuf *d_buf;

	list_remove_head(rb_list, d_buf, struct lpfc_dmabuf, list);
	if (!d_buf)
		return NULL;
	return container_of(d_buf, struct hbq_dmabuf, dbuf);
}

/**
 * lpfc_sli_rqbuf_get - Remove the first dma buffer off of an RQ list
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 *
 * This function removes the first RQ buffer on an RQ buffer list and returns a
 * pointer to that buffer. If it finds no buffers on the list it returns NULL.
 **/
static struct rqb_dmabuf *
lpfc_sli_rqbuf_get(struct lpfc_hba *phba, struct lpfc_queue *hrq)
{
	struct lpfc_dmabuf *h_buf;
	struct lpfc_rqb *rqbp;

	rqbp = hrq->rqbp;
	list_remove_head(&rqbp->rqb_buffer_list, h_buf,
			 struct lpfc_dmabuf, list);
	if (!h_buf)
		return NULL;
	rqbp->buffer_count--;
	return container_of(h_buf, struct rqb_dmabuf, hbuf);
}

/**
 * lpfc_sli_hbqbuf_find - Find the hbq buffer associated with a tag
 * @phba: Pointer to HBA context object.
 * @tag: Tag of the hbq buffer.
 *
 * This function searches for the hbq buffer associated with the given tag in
 * the hbq buffer list. If it finds the hbq buffer, it returns the hbq_buffer
 * otherwise it returns NULL.
 **/
static struct hbq_dmabuf *
lpfc_sli_hbqbuf_find(struct lpfc_hba *phba, uint32_t tag)
{
	struct lpfc_dmabuf *d_buf;
	struct hbq_dmabuf *hbq_buf;
	uint32_t hbqno;

	hbqno = tag >> 16;
	if (hbqno >= LPFC_MAX_HBQS)
		return NULL;

	spin_lock_irq(&phba->hbalock);
	list_for_each_entry(d_buf, &phba->hbqs[hbqno].hbq_buffer_list, list) {
		hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
		if (hbq_buf->tag == tag) {
			spin_unlock_irq(&phba->hbalock);
			return hbq_buf;
		}
	}
	spin_unlock_irq(&phba->hbalock);
	lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_VPORT,
			"1803 Bad hbq tag. Data: x%x x%x\n",
			tag, phba->hbqs[tag >> 16].buffer_count);
	return NULL;
}

/**
 * lpfc_sli_free_hbq - Give back the hbq buffer to firmware
 * @phba: Pointer to HBA context object.
 * @hbq_buffer: Pointer to HBQ buffer.
 *
 * This function is called with hbalock. This function gives back
 * the hbq buffer to firmware. If the HBQ does not have space to
 * post the buffer, it will free the buffer.
 **/
void
lpfc_sli_free_hbq(struct lpfc_hba *phba, struct hbq_dmabuf *hbq_buffer)
{
	uint32_t hbqno;

	if (hbq_buffer) {
		hbqno = hbq_buffer->tag >> 16;
		if (lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buffer))
			(phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
	}
}

/**
 * lpfc_sli_chk_mbx_command - Check if the mailbox is a legitimate mailbox
 * @mbxCommand: mailbox command code.
 *
 * This function is called by the mailbox event handler function to verify
 * that the completed mailbox command is a legitimate mailbox command. If the
 * completed mailbox is not known to the function, it will return MBX_SHUTDOWN
 * and the mailbox event handler will take the HBA offline.
 **/
static int
lpfc_sli_chk_mbx_command(uint8_t mbxCommand)
{
	uint8_t ret;

	switch (mbxCommand) {
	case MBX_LOAD_SM:
	case MBX_READ_NV:
	case MBX_WRITE_NV:
	case MBX_WRITE_VPARMS:
	case MBX_RUN_BIU_DIAG:
	case MBX_INIT_LINK:
	case MBX_DOWN_LINK:
	case MBX_CONFIG_LINK:
	case MBX_CONFIG_RING:
	case MBX_RESET_RING:
	case MBX_READ_CONFIG:
	case MBX_READ_RCONFIG:
	case MBX_READ_SPARM:
	case MBX_READ_STATUS:
	case MBX_READ_RPI:
	case MBX_READ_XRI:
	case MBX_READ_REV:
	case MBX_READ_LNK_STAT:
	case MBX_REG_LOGIN:
	case MBX_UNREG_LOGIN:
	case MBX_CLEAR_LA:
	case MBX_DUMP_MEMORY:
	case MBX_DUMP_CONTEXT:
	case MBX_RUN_DIAGS:
	case MBX_RESTART:
	case MBX_UPDATE_CFG:
	case MBX_DOWN_LOAD:
	case MBX_DEL_LD_ENTRY:
	case MBX_RUN_PROGRAM:
	case MBX_SET_MASK:
	case MBX_SET_VARIABLE:
	case MBX_UNREG_D_ID:
	case MBX_KILL_BOARD:
	case MBX_CONFIG_FARP:
	case MBX_BEACON:
	case MBX_LOAD_AREA:
	case MBX_RUN_BIU_DIAG64:
	case MBX_CONFIG_PORT:
	case MBX_READ_SPARM64:
	case MBX_READ_RPI64:
	case MBX_REG_LOGIN64:
	case MBX_READ_TOPOLOGY:
	case MBX_WRITE_WWN:
	case MBX_SET_DEBUG:
	case MBX_LOAD_EXP_ROM:
	case MBX_ASYNCEVT_ENABLE:
	case MBX_REG_VPI:
	case MBX_UNREG_VPI:
	case MBX_HEARTBEAT:
	case MBX_PORT_CAPABILITIES:
	case MBX_PORT_IOV_CONTROL:
	case MBX_SLI4_CONFIG:
	case MBX_SLI4_REQ_FTRS:
	case MBX_REG_FCFI:
	case MBX_UNREG_FCFI:
	case MBX_REG_VFI:
	case MBX_UNREG_VFI:
	case MBX_INIT_VPI:
	case MBX_INIT_VFI:
	case MBX_RESUME_RPI:
	case MBX_READ_EVENT_LOG_STATUS:
	case MBX_READ_EVENT_LOG:
	case MBX_SECURITY_MGMT:
	case MBX_AUTH_PORT:
	case MBX_ACCESS_VDATA:
		ret = mbxCommand;
		break;
	default:
		ret = MBX_SHUTDOWN;
		break;
	}
	return ret;
}

/**
 * lpfc_sli_wake_mbox_wait - lpfc_sli_issue_mbox_wait mbox completion handler
 * @phba: Pointer to HBA context object.
 * @pmboxq: Pointer to mailbox command.
 *
 * This is completion handler function for mailbox commands issued from
 * lpfc_sli_issue_mbox_wait function. This function is called by the
 * mailbox event handler function with no lock held. This function
 * will wake up thread waiting on the wait queue pointed by context1
 * of the mailbox.
 **/
void
lpfc_sli_wake_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
{
	unsigned long drvr_flag;
	struct completion *pmbox_done;

	/*
	 * If pmbox_done is empty, the driver thread gave up waiting and
	 * continued running.
	 */
	pmboxq->mbox_flag |= LPFC_MBX_WAKE;
	spin_lock_irqsave(&phba->hbalock, drvr_flag);
	pmbox_done = (struct completion *)pmboxq->context3;
	if (pmbox_done)
		complete(pmbox_done);
	spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
	return;
}


/**
 * lpfc_sli_def_mbox_cmpl - Default mailbox completion handler
 * @phba: Pointer to HBA context object.
 * @pmb: Pointer to mailbox object.
 *
 * This function is the default mailbox completion handler. It
 * frees the memory resources associated with the completed mailbox
 * command. If the completed command is a REG_LOGIN mailbox command,
 * this function will issue a UREG_LOGIN to re-claim the RPI.
 **/
void
lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
{
	struct lpfc_vport  *vport = pmb->vport;
	struct lpfc_dmabuf *mp;
	struct lpfc_nodelist *ndlp;
	struct Scsi_Host *shost;
	uint16_t rpi, vpi;
	int rc;

	mp = (struct lpfc_dmabuf *)(pmb->ctx_buf);

	if (mp) {
		lpfc_mbuf_free(phba, mp->virt, mp->phys);
		kfree(mp);
	}

	/*
	 * If a REG_LOGIN succeeded  after node is destroyed or node
	 * is in re-discovery driver need to cleanup the RPI.
	 */
	if (!(phba->pport->load_flag & FC_UNLOADING) &&
	    pmb->u.mb.mbxCommand == MBX_REG_LOGIN64 &&
	    !pmb->u.mb.mbxStatus) {
		rpi = pmb->u.mb.un.varWords[0];
		vpi = pmb->u.mb.un.varRegLogin.vpi;
		lpfc_unreg_login(phba, vpi, rpi, pmb);
		pmb->vport = vport;
		pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
		rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
		if (rc != MBX_NOT_FINISHED)
			return;
	}

	if ((pmb->u.mb.mbxCommand == MBX_REG_VPI) &&
		!(phba->pport->load_flag & FC_UNLOADING) &&
		!pmb->u.mb.mbxStatus) {
		shost = lpfc_shost_from_vport(vport);
		spin_lock_irq(shost->host_lock);
		vport->vpi_state |= LPFC_VPI_REGISTERED;
		vport->fc_flag &= ~FC_VPORT_NEEDS_REG_VPI;
		spin_unlock_irq(shost->host_lock);
	}

	if (pmb->u.mb.mbxCommand == MBX_REG_LOGIN64) {
		ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
		lpfc_nlp_put(ndlp);
		pmb->ctx_buf = NULL;
		pmb->ctx_ndlp = NULL;
	}

	if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) {
		ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;

		/* Check to see if there are any deferred events to process */
		if (ndlp) {
			lpfc_printf_vlog(
				vport,
				KERN_INFO, LOG_MBOX | LOG_DISCOVERY,
				"1438 UNREG cmpl deferred mbox x%x "
				"on NPort x%x Data: x%x x%x %p\n",
				ndlp->nlp_rpi, ndlp->nlp_DID,
				ndlp->nlp_flag, ndlp->nlp_defer_did, ndlp);

			if ((ndlp->nlp_flag & NLP_UNREG_INP) &&
			    (ndlp->nlp_defer_did != NLP_EVT_NOTHING_PENDING)) {
				ndlp->nlp_flag &= ~NLP_UNREG_INP;
				ndlp->nlp_defer_did = NLP_EVT_NOTHING_PENDING;
				lpfc_issue_els_plogi(vport, ndlp->nlp_DID, 0);
			} else {
				ndlp->nlp_flag &= ~NLP_UNREG_INP;
			}
			pmb->ctx_ndlp = NULL;
		}
	}

	/* Check security permission status on INIT_LINK mailbox command */
	if ((pmb->u.mb.mbxCommand == MBX_INIT_LINK) &&
	    (pmb->u.mb.mbxStatus == MBXERR_SEC_NO_PERMISSION))
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"2860 SLI authentication is required "
				"for INIT_LINK but has not done yet\n");

	if (bf_get(lpfc_mqe_command, &pmb->u.mqe) == MBX_SLI4_CONFIG)
		lpfc_sli4_mbox_cmd_free(phba, pmb);
	else
		mempool_free(pmb, phba->mbox_mem_pool);
}
 /**
 * lpfc_sli4_unreg_rpi_cmpl_clr - mailbox completion handler
 * @phba: Pointer to HBA context object.
 * @pmb: Pointer to mailbox object.
 *
 * This function is the unreg rpi mailbox completion handler. It
 * frees the memory resources associated with the completed mailbox
 * command. An additional refrenece is put on the ndlp to prevent
 * lpfc_nlp_release from freeing the rpi bit in the bitmask before
 * the unreg mailbox command completes, this routine puts the
 * reference back.
 *
 **/
void
lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
{
	struct lpfc_vport  *vport = pmb->vport;
	struct lpfc_nodelist *ndlp;

	ndlp = pmb->ctx_ndlp;
	if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) {
		if (phba->sli_rev == LPFC_SLI_REV4 &&
		    (bf_get(lpfc_sli_intf_if_type,
		     &phba->sli4_hba.sli_intf) >=
		     LPFC_SLI_INTF_IF_TYPE_2)) {
			if (ndlp) {
				lpfc_printf_vlog(
					vport, KERN_INFO, LOG_MBOX | LOG_SLI,
					 "0010 UNREG_LOGIN vpi:%x "
					 "rpi:%x DID:%x defer x%x flg x%x "
					 "map:%x %p\n",
					 vport->vpi, ndlp->nlp_rpi,
					 ndlp->nlp_DID, ndlp->nlp_defer_did,
					 ndlp->nlp_flag,
					 ndlp->nlp_usg_map, ndlp);
				ndlp->nlp_flag &= ~NLP_LOGO_ACC;
				lpfc_nlp_put(ndlp);

				/* Check to see if there are any deferred
				 * events to process
				 */
				if ((ndlp->nlp_flag & NLP_UNREG_INP) &&
				    (ndlp->nlp_defer_did !=
				    NLP_EVT_NOTHING_PENDING)) {
					lpfc_printf_vlog(
						vport, KERN_INFO, LOG_DISCOVERY,
						"4111 UNREG cmpl deferred "
						"clr x%x on "
						"NPort x%x Data: x%x %p\n",
						ndlp->nlp_rpi, ndlp->nlp_DID,
						ndlp->nlp_defer_did, ndlp);
					ndlp->nlp_flag &= ~NLP_UNREG_INP;
					ndlp->nlp_defer_did =
						NLP_EVT_NOTHING_PENDING;
					lpfc_issue_els_plogi(
						vport, ndlp->nlp_DID, 0);
				} else {
					ndlp->nlp_flag &= ~NLP_UNREG_INP;
				}
			}
		}
	}

	mempool_free(pmb, phba->mbox_mem_pool);
}

/**
 * lpfc_sli_handle_mb_event - Handle mailbox completions from firmware
 * @phba: Pointer to HBA context object.
 *
 * This function is called with no lock held. This function processes all
 * the completed mailbox commands and gives it to upper layers. The interrupt
 * service routine processes mailbox completion interrupt and adds completed
 * mailbox commands to the mboxq_cmpl queue and signals the worker thread.
 * Worker thread call lpfc_sli_handle_mb_event, which will return the
 * completed mailbox commands in mboxq_cmpl queue to the upper layers. This
 * function returns the mailbox commands to the upper layer by calling the
 * completion handler function of each mailbox.
 **/
int
lpfc_sli_handle_mb_event(struct lpfc_hba *phba)
{
	MAILBOX_t *pmbox;
	LPFC_MBOXQ_t *pmb;
	int rc;
	LIST_HEAD(cmplq);

	phba->sli.slistat.mbox_event++;

	/* Get all completed mailboxe buffers into the cmplq */
	spin_lock_irq(&phba->hbalock);
	list_splice_init(&phba->sli.mboxq_cmpl, &cmplq);
	spin_unlock_irq(&phba->hbalock);

	/* Get a Mailbox buffer to setup mailbox commands for callback */
	do {
		list_remove_head(&cmplq, pmb, LPFC_MBOXQ_t, list);
		if (pmb == NULL)
			break;

		pmbox = &pmb->u.mb;

		if (pmbox->mbxCommand != MBX_HEARTBEAT) {
			if (pmb->vport) {
				lpfc_debugfs_disc_trc(pmb->vport,
					LPFC_DISC_TRC_MBOX_VPORT,
					"MBOX cmpl vport: cmd:x%x mb:x%x x%x",
					(uint32_t)pmbox->mbxCommand,
					pmbox->un.varWords[0],
					pmbox->un.varWords[1]);
			}
			else {
				lpfc_debugfs_disc_trc(phba->pport,
					LPFC_DISC_TRC_MBOX,
					"MBOX cmpl:       cmd:x%x mb:x%x x%x",
					(uint32_t)pmbox->mbxCommand,
					pmbox->un.varWords[0],
					pmbox->un.varWords[1]);
			}
		}

		/*
		 * It is a fatal error if unknown mbox command completion.
		 */
		if (lpfc_sli_chk_mbx_command(pmbox->mbxCommand) ==
		    MBX_SHUTDOWN) {
			/* Unknown mailbox command compl */
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"(%d):0323 Unknown Mailbox command "
					"x%x (x%x/x%x) Cmpl\n",
					pmb->vport ? pmb->vport->vpi : 0,
					pmbox->mbxCommand,
					lpfc_sli_config_mbox_subsys_get(phba,
									pmb),
					lpfc_sli_config_mbox_opcode_get(phba,
									pmb));
			phba->link_state = LPFC_HBA_ERROR;
			phba->work_hs = HS_FFER3;
			lpfc_handle_eratt(phba);
			continue;
		}

		if (pmbox->mbxStatus) {
			phba->sli.slistat.mbox_stat_err++;
			if (pmbox->mbxStatus == MBXERR_NO_RESOURCES) {
				/* Mbox cmd cmpl error - RETRYing */
				lpfc_printf_log(phba, KERN_INFO,
					LOG_MBOX | LOG_SLI,
					"(%d):0305 Mbox cmd cmpl "
					"error - RETRYing Data: x%x "
					"(x%x/x%x) x%x x%x x%x\n",
					pmb->vport ? pmb->vport->vpi : 0,
					pmbox->mbxCommand,
					lpfc_sli_config_mbox_subsys_get(phba,
									pmb),
					lpfc_sli_config_mbox_opcode_get(phba,
									pmb),
					pmbox->mbxStatus,
					pmbox->un.varWords[0],
					pmb->vport->port_state);
				pmbox->mbxStatus = 0;
				pmbox->mbxOwner = OWN_HOST;
				rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
				if (rc != MBX_NOT_FINISHED)
					continue;
			}
		}

		/* Mailbox cmd <cmd> Cmpl <cmpl> */
		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
				"(%d):0307 Mailbox cmd x%x (x%x/x%x) Cmpl x%p "
				"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x "
				"x%x x%x x%x\n",
				pmb->vport ? pmb->vport->vpi : 0,
				pmbox->mbxCommand,
				lpfc_sli_config_mbox_subsys_get(phba, pmb),
				lpfc_sli_config_mbox_opcode_get(phba, pmb),
				pmb->mbox_cmpl,
				*((uint32_t *) pmbox),
				pmbox->un.varWords[0],
				pmbox->un.varWords[1],
				pmbox->un.varWords[2],
				pmbox->un.varWords[3],
				pmbox->un.varWords[4],
				pmbox->un.varWords[5],
				pmbox->un.varWords[6],
				pmbox->un.varWords[7],
				pmbox->un.varWords[8],
				pmbox->un.varWords[9],
				pmbox->un.varWords[10]);

		if (pmb->mbox_cmpl)
			pmb->mbox_cmpl(phba,pmb);
	} while (1);
	return 0;
}

/**
 * lpfc_sli_get_buff - Get the buffer associated with the buffer tag
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @tag: buffer tag.
 *
 * This function is called with no lock held. When QUE_BUFTAG_BIT bit
 * is set in the tag the buffer is posted for a particular exchange,
 * the function will return the buffer without replacing the buffer.
 * If the buffer is for unsolicited ELS or CT traffic, this function
 * returns the buffer and also posts another buffer to the firmware.
 **/
static struct lpfc_dmabuf *
lpfc_sli_get_buff(struct lpfc_hba *phba,
		  struct lpfc_sli_ring *pring,
		  uint32_t tag)
{
	struct hbq_dmabuf *hbq_entry;

	if (tag & QUE_BUFTAG_BIT)
		return lpfc_sli_ring_taggedbuf_get(phba, pring, tag);
	hbq_entry = lpfc_sli_hbqbuf_find(phba, tag);
	if (!hbq_entry)
		return NULL;
	return &hbq_entry->dbuf;
}

/**
 * lpfc_complete_unsol_iocb - Complete an unsolicited sequence
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @saveq: Pointer to the iocbq struct representing the sequence starting frame.
 * @fch_r_ctl: the r_ctl for the first frame of the sequence.
 * @fch_type: the type for the first frame of the sequence.
 *
 * This function is called with no lock held. This function uses the r_ctl and
 * type of the received sequence to find the correct callback function to call
 * to process the sequence.
 **/
static int
lpfc_complete_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			 struct lpfc_iocbq *saveq, uint32_t fch_r_ctl,
			 uint32_t fch_type)
{
	int i;

	switch (fch_type) {
	case FC_TYPE_NVME:
		lpfc_nvmet_unsol_ls_event(phba, pring, saveq);
		return 1;
	default:
		break;
	}

	/* unSolicited Responses */
	if (pring->prt[0].profile) {
		if (pring->prt[0].lpfc_sli_rcv_unsol_event)
			(pring->prt[0].lpfc_sli_rcv_unsol_event) (phba, pring,
									saveq);
		return 1;
	}
	/* We must search, based on rctl / type
	   for the right routine */
	for (i = 0; i < pring->num_mask; i++) {
		if ((pring->prt[i].rctl == fch_r_ctl) &&
		    (pring->prt[i].type == fch_type)) {
			if (pring->prt[i].lpfc_sli_rcv_unsol_event)
				(pring->prt[i].lpfc_sli_rcv_unsol_event)
						(phba, pring, saveq);
			return 1;
		}
	}
	return 0;
}

/**
 * lpfc_sli_process_unsol_iocb - Unsolicited iocb handler
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @saveq: Pointer to the unsolicited iocb.
 *
 * This function is called with no lock held by the ring event handler
 * when there is an unsolicited iocb posted to the response ring by the
 * firmware. This function gets the buffer associated with the iocbs
 * and calls the event handler for the ring. This function handles both
 * qring buffers and hbq buffers.
 * When the function returns 1 the caller can free the iocb object otherwise
 * upper layer functions will free the iocb objects.
 **/
static int
lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			    struct lpfc_iocbq *saveq)
{
	IOCB_t           * irsp;
	WORD5            * w5p;
	uint32_t           Rctl, Type;
	struct lpfc_iocbq *iocbq;
	struct lpfc_dmabuf *dmzbuf;

	irsp = &(saveq->iocb);

	if (irsp->ulpCommand == CMD_ASYNC_STATUS) {
		if (pring->lpfc_sli_rcv_async_status)
			pring->lpfc_sli_rcv_async_status(phba, pring, saveq);
		else
			lpfc_printf_log(phba,
					KERN_WARNING,
					LOG_SLI,
					"0316 Ring %d handler: unexpected "
					"ASYNC_STATUS iocb received evt_code "
					"0x%x\n",
					pring->ringno,
					irsp->un.asyncstat.evt_code);
		return 1;
	}

	if ((irsp->ulpCommand == CMD_IOCB_RET_XRI64_CX) &&
		(phba->sli3_options & LPFC_SLI3_HBQ_ENABLED)) {
		if (irsp->ulpBdeCount > 0) {
			dmzbuf = lpfc_sli_get_buff(phba, pring,
					irsp->un.ulpWord[3]);
			lpfc_in_buf_free(phba, dmzbuf);
		}

		if (irsp->ulpBdeCount > 1) {
			dmzbuf = lpfc_sli_get_buff(phba, pring,
					irsp->unsli3.sli3Words[3]);
			lpfc_in_buf_free(phba, dmzbuf);
		}

		if (irsp->ulpBdeCount > 2) {
			dmzbuf = lpfc_sli_get_buff(phba, pring,
				irsp->unsli3.sli3Words[7]);
			lpfc_in_buf_free(phba, dmzbuf);
		}

		return 1;
	}

	if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
		if (irsp->ulpBdeCount != 0) {
			saveq->context2 = lpfc_sli_get_buff(phba, pring,
						irsp->un.ulpWord[3]);
			if (!saveq->context2)
				lpfc_printf_log(phba,
					KERN_ERR,
					LOG_SLI,
					"0341 Ring %d Cannot find buffer for "
					"an unsolicited iocb. tag 0x%x\n",
					pring->ringno,
					irsp->un.ulpWord[3]);
		}
		if (irsp->ulpBdeCount == 2) {
			saveq->context3 = lpfc_sli_get_buff(phba, pring,
						irsp->unsli3.sli3Words[7]);
			if (!saveq->context3)
				lpfc_printf_log(phba,
					KERN_ERR,
					LOG_SLI,
					"0342 Ring %d Cannot find buffer for an"
					" unsolicited iocb. tag 0x%x\n",
					pring->ringno,
					irsp->unsli3.sli3Words[7]);
		}
		list_for_each_entry(iocbq, &saveq->list, list) {
			irsp = &(iocbq->iocb);
			if (irsp->ulpBdeCount != 0) {
				iocbq->context2 = lpfc_sli_get_buff(phba, pring,
							irsp->un.ulpWord[3]);
				if (!iocbq->context2)
					lpfc_printf_log(phba,
						KERN_ERR,
						LOG_SLI,
						"0343 Ring %d Cannot find "
						"buffer for an unsolicited iocb"
						". tag 0x%x\n", pring->ringno,
						irsp->un.ulpWord[3]);
			}
			if (irsp->ulpBdeCount == 2) {
				iocbq->context3 = lpfc_sli_get_buff(phba, pring,
						irsp->unsli3.sli3Words[7]);
				if (!iocbq->context3)
					lpfc_printf_log(phba,
						KERN_ERR,
						LOG_SLI,
						"0344 Ring %d Cannot find "
						"buffer for an unsolicited "
						"iocb. tag 0x%x\n",
						pring->ringno,
						irsp->unsli3.sli3Words[7]);
			}
		}
	}
	if (irsp->ulpBdeCount != 0 &&
	    (irsp->ulpCommand == CMD_IOCB_RCV_CONT64_CX ||
	     irsp->ulpStatus == IOSTAT_INTERMED_RSP)) {
		int found = 0;

		/* search continue save q for same XRI */
		list_for_each_entry(iocbq, &pring->iocb_continue_saveq, clist) {
			if (iocbq->iocb.unsli3.rcvsli3.ox_id ==
				saveq->iocb.unsli3.rcvsli3.ox_id) {
				list_add_tail(&saveq->list, &iocbq->list);
				found = 1;
				break;
			}
		}
		if (!found)
			list_add_tail(&saveq->clist,
				      &pring->iocb_continue_saveq);
		if (saveq->iocb.ulpStatus != IOSTAT_INTERMED_RSP) {
			list_del_init(&iocbq->clist);
			saveq = iocbq;
			irsp = &(saveq->iocb);
		} else
			return 0;
	}
	if ((irsp->ulpCommand == CMD_RCV_ELS_REQ64_CX) ||
	    (irsp->ulpCommand == CMD_RCV_ELS_REQ_CX) ||
	    (irsp->ulpCommand == CMD_IOCB_RCV_ELS64_CX)) {
		Rctl = FC_RCTL_ELS_REQ;
		Type = FC_TYPE_ELS;
	} else {
		w5p = (WORD5 *)&(saveq->iocb.un.ulpWord[5]);
		Rctl = w5p->hcsw.Rctl;
		Type = w5p->hcsw.Type;

		/* Firmware Workaround */
		if ((Rctl == 0) && (pring->ringno == LPFC_ELS_RING) &&
			(irsp->ulpCommand == CMD_RCV_SEQUENCE64_CX ||
			 irsp->ulpCommand == CMD_IOCB_RCV_SEQ64_CX)) {
			Rctl = FC_RCTL_ELS_REQ;
			Type = FC_TYPE_ELS;
			w5p->hcsw.Rctl = Rctl;
			w5p->hcsw.Type = Type;
		}
	}

	if (!lpfc_complete_unsol_iocb(phba, pring, saveq, Rctl, Type))
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"0313 Ring %d handler: unexpected Rctl x%x "
				"Type x%x received\n",
				pring->ringno, Rctl, Type);

	return 1;
}

/**
 * lpfc_sli_iocbq_lookup - Find command iocb for the given response iocb
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @prspiocb: Pointer to response iocb object.
 *
 * This function looks up the iocb_lookup table to get the command iocb
 * corresponding to the given response iocb using the iotag of the
 * response iocb. The driver calls this function with the hbalock held
 * for SLI3 ports or the ring lock held for SLI4 ports.
 * This function returns the command iocb object if it finds the command
 * iocb else returns NULL.
 **/
static struct lpfc_iocbq *
lpfc_sli_iocbq_lookup(struct lpfc_hba *phba,
		      struct lpfc_sli_ring *pring,
		      struct lpfc_iocbq *prspiocb)
{
	struct lpfc_iocbq *cmd_iocb = NULL;
	uint16_t iotag;
	spinlock_t *temp_lock = NULL;
	unsigned long iflag = 0;

	if (phba->sli_rev == LPFC_SLI_REV4)
		temp_lock = &pring->ring_lock;
	else
		temp_lock = &phba->hbalock;

	spin_lock_irqsave(temp_lock, iflag);
	iotag = prspiocb->iocb.ulpIoTag;

	if (iotag != 0 && iotag <= phba->sli.last_iotag) {
		cmd_iocb = phba->sli.iocbq_lookup[iotag];
		if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) {
			/* remove from txcmpl queue list */
			list_del_init(&cmd_iocb->list);
			cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
			pring->txcmplq_cnt--;
			spin_unlock_irqrestore(temp_lock, iflag);
			return cmd_iocb;
		}
	}

	spin_unlock_irqrestore(temp_lock, iflag);
	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
			"0317 iotag x%x is out of "
			"range: max iotag x%x wd0 x%x\n",
			iotag, phba->sli.last_iotag,
			*(((uint32_t *) &prspiocb->iocb) + 7));
	return NULL;
}

/**
 * lpfc_sli_iocbq_lookup_by_tag - Find command iocb for the iotag
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @iotag: IOCB tag.
 *
 * This function looks up the iocb_lookup table to get the command iocb
 * corresponding to the given iotag. The driver calls this function with
 * the ring lock held because this function is an SLI4 port only helper.
 * This function returns the command iocb object if it finds the command
 * iocb else returns NULL.
 **/
static struct lpfc_iocbq *
lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba,
			     struct lpfc_sli_ring *pring, uint16_t iotag)
{
	struct lpfc_iocbq *cmd_iocb = NULL;
	spinlock_t *temp_lock = NULL;
	unsigned long iflag = 0;

	if (phba->sli_rev == LPFC_SLI_REV4)
		temp_lock = &pring->ring_lock;
	else
		temp_lock = &phba->hbalock;

	spin_lock_irqsave(temp_lock, iflag);
	if (iotag != 0 && iotag <= phba->sli.last_iotag) {
		cmd_iocb = phba->sli.iocbq_lookup[iotag];
		if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) {
			/* remove from txcmpl queue list */
			list_del_init(&cmd_iocb->list);
			cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
			pring->txcmplq_cnt--;
			spin_unlock_irqrestore(temp_lock, iflag);
			return cmd_iocb;
		}
	}

	spin_unlock_irqrestore(temp_lock, iflag);
	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
			"0372 iotag x%x lookup error: max iotag (x%x) "
			"iocb_flag x%x\n",
			iotag, phba->sli.last_iotag,
			cmd_iocb ? cmd_iocb->iocb_flag : 0xffff);
	return NULL;
}

/**
 * lpfc_sli_process_sol_iocb - process solicited iocb completion
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @saveq: Pointer to the response iocb to be processed.
 *
 * This function is called by the ring event handler for non-fcp
 * rings when there is a new response iocb in the response ring.
 * The caller is not required to hold any locks. This function
 * gets the command iocb associated with the response iocb and
 * calls the completion handler for the command iocb. If there
 * is no completion handler, the function will free the resources
 * associated with command iocb. If the response iocb is for
 * an already aborted command iocb, the status of the completion
 * is changed to IOSTAT_LOCAL_REJECT/IOERR_SLI_ABORTED.
 * This function always returns 1.
 **/
static int
lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			  struct lpfc_iocbq *saveq)
{
	struct lpfc_iocbq *cmdiocbp;
	int rc = 1;
	unsigned long iflag;

	cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring, saveq);
	if (cmdiocbp) {
		if (cmdiocbp->iocb_cmpl) {
			/*
			 * If an ELS command failed send an event to mgmt
			 * application.
			 */
			if (saveq->iocb.ulpStatus &&
			     (pring->ringno == LPFC_ELS_RING) &&
			     (cmdiocbp->iocb.ulpCommand ==
				CMD_ELS_REQUEST64_CR))
				lpfc_send_els_failure_event(phba,
					cmdiocbp, saveq);

			/*
			 * Post all ELS completions to the worker thread.
			 * All other are passed to the completion callback.
			 */
			if (pring->ringno == LPFC_ELS_RING) {
				if ((phba->sli_rev < LPFC_SLI_REV4) &&
				    (cmdiocbp->iocb_flag &
							LPFC_DRIVER_ABORTED)) {
					spin_lock_irqsave(&phba->hbalock,
							  iflag);
					cmdiocbp->iocb_flag &=
						~LPFC_DRIVER_ABORTED;
					spin_unlock_irqrestore(&phba->hbalock,
							       iflag);
					saveq->iocb.ulpStatus =
						IOSTAT_LOCAL_REJECT;
					saveq->iocb.un.ulpWord[4] =
						IOERR_SLI_ABORTED;

					/* Firmware could still be in progress
					 * of DMAing payload, so don't free data
					 * buffer till after a hbeat.
					 */
					spin_lock_irqsave(&phba->hbalock,
							  iflag);
					saveq->iocb_flag |= LPFC_DELAY_MEM_FREE;
					spin_unlock_irqrestore(&phba->hbalock,
							       iflag);
				}
				if (phba->sli_rev == LPFC_SLI_REV4) {
					if (saveq->iocb_flag &
					    LPFC_EXCHANGE_BUSY) {
						/* Set cmdiocb flag for the
						 * exchange busy so sgl (xri)
						 * will not be released until
						 * the abort xri is received
						 * from hba.
						 */
						spin_lock_irqsave(
							&phba->hbalock, iflag);
						cmdiocbp->iocb_flag |=
							LPFC_EXCHANGE_BUSY;
						spin_unlock_irqrestore(
							&phba->hbalock, iflag);
					}
					if (cmdiocbp->iocb_flag &
					    LPFC_DRIVER_ABORTED) {
						/*
						 * Clear LPFC_DRIVER_ABORTED
						 * bit in case it was driver
						 * initiated abort.
						 */
						spin_lock_irqsave(
							&phba->hbalock, iflag);
						cmdiocbp->iocb_flag &=
							~LPFC_DRIVER_ABORTED;
						spin_unlock_irqrestore(
							&phba->hbalock, iflag);
						cmdiocbp->iocb.ulpStatus =
							IOSTAT_LOCAL_REJECT;
						cmdiocbp->iocb.un.ulpWord[4] =
							IOERR_ABORT_REQUESTED;
						/*
						 * For SLI4, irsiocb contains
						 * NO_XRI in sli_xritag, it
						 * shall not affect releasing
						 * sgl (xri) process.
						 */
						saveq->iocb.ulpStatus =
							IOSTAT_LOCAL_REJECT;
						saveq->iocb.un.ulpWord[4] =
							IOERR_SLI_ABORTED;
						spin_lock_irqsave(
							&phba->hbalock, iflag);
						saveq->iocb_flag |=
							LPFC_DELAY_MEM_FREE;
						spin_unlock_irqrestore(
							&phba->hbalock, iflag);
					}
				}
			}
			(cmdiocbp->iocb_cmpl) (phba, cmdiocbp, saveq);
		} else
			lpfc_sli_release_iocbq(phba, cmdiocbp);
	} else {
		/*
		 * Unknown initiating command based on the response iotag.
		 * This could be the case on the ELS ring because of
		 * lpfc_els_abort().
		 */
		if (pring->ringno != LPFC_ELS_RING) {
			/*
			 * Ring <ringno> handler: unexpected completion IoTag
			 * <IoTag>
			 */
			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
					 "0322 Ring %d handler: "
					 "unexpected completion IoTag x%x "
					 "Data: x%x x%x x%x x%x\n",
					 pring->ringno,
					 saveq->iocb.ulpIoTag,
					 saveq->iocb.ulpStatus,
					 saveq->iocb.un.ulpWord[4],
					 saveq->iocb.ulpCommand,
					 saveq->iocb.ulpContext);
		}
	}

	return rc;
}

/**
 * lpfc_sli_rsp_pointers_error - Response ring pointer error handler
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called from the iocb ring event handlers when
 * put pointer is ahead of the get pointer for a ring. This function signal
 * an error attention condition to the worker thread and the worker
 * thread will transition the HBA to offline state.
 **/
static void
lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
	/*
	 * Ring <ringno> handler: portRspPut <portRspPut> is bigger than
	 * rsp ring <portRspMax>
	 */
	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
			"0312 Ring %d handler: portRspPut %d "
			"is bigger than rsp ring %d\n",
			pring->ringno, le32_to_cpu(pgp->rspPutInx),
			pring->sli.sli3.numRiocb);

	phba->link_state = LPFC_HBA_ERROR;

	/*
	 * All error attention handlers are posted to
	 * worker thread
	 */
	phba->work_ha |= HA_ERATT;
	phba->work_hs = HS_FFER3;

	lpfc_worker_wake_up(phba);

	return;
}

/**
 * lpfc_poll_eratt - Error attention polling timer timeout handler
 * @ptr: Pointer to address of HBA context object.
 *
 * This function is invoked by the Error Attention polling timer when the
 * timer times out. It will check the SLI Error Attention register for
 * possible attention events. If so, it will post an Error Attention event
 * and wake up worker thread to process it. Otherwise, it will set up the
 * Error Attention polling timer for the next poll.
 **/
void lpfc_poll_eratt(struct timer_list *t)
{
	struct lpfc_hba *phba;
	uint32_t eratt = 0;
	uint64_t sli_intr, cnt;

	phba = from_timer(phba, t, eratt_poll);

	/* Here we will also keep track of interrupts per sec of the hba */
	sli_intr = phba->sli.slistat.sli_intr;

	if (phba->sli.slistat.sli_prev_intr > sli_intr)
		cnt = (((uint64_t)(-1) - phba->sli.slistat.sli_prev_intr) +
			sli_intr);
	else
		cnt = (sli_intr - phba->sli.slistat.sli_prev_intr);

	/* 64-bit integer division not supported on 32-bit x86 - use do_div */
	do_div(cnt, phba->eratt_poll_interval);
	phba->sli.slistat.sli_ips = cnt;

	phba->sli.slistat.sli_prev_intr = sli_intr;

	/* Check chip HA register for error event */
	eratt = lpfc_sli_check_eratt(phba);

	if (eratt)
		/* Tell the worker thread there is work to do */
		lpfc_worker_wake_up(phba);
	else
		/* Restart the timer for next eratt poll */
		mod_timer(&phba->eratt_poll,
			  jiffies +
			  msecs_to_jiffies(1000 * phba->eratt_poll_interval));
	return;
}


/**
 * lpfc_sli_handle_fast_ring_event - Handle ring events on FCP ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mask: Host attention register mask for this ring.
 *
 * This function is called from the interrupt context when there is a ring
 * event for the fcp ring. The caller does not hold any lock.
 * The function processes each response iocb in the response ring until it
 * finds an iocb with LE bit set and chains all the iocbs up to the iocb with
 * LE bit set. The function will call the completion handler of the command iocb
 * if the response iocb indicates a completion for a command iocb or it is
 * an abort completion. The function will call lpfc_sli_process_unsol_iocb
 * function if this is an unsolicited iocb.
 * This routine presumes LPFC_FCP_RING handling and doesn't bother
 * to check it explicitly.
 */
int
lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
				struct lpfc_sli_ring *pring, uint32_t mask)
{
	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
	IOCB_t *irsp = NULL;
	IOCB_t *entry = NULL;
	struct lpfc_iocbq *cmdiocbq = NULL;
	struct lpfc_iocbq rspiocbq;
	uint32_t status;
	uint32_t portRspPut, portRspMax;
	int rc = 1;
	lpfc_iocb_type type;
	unsigned long iflag;
	uint32_t rsp_cmpl = 0;

	spin_lock_irqsave(&phba->hbalock, iflag);
	pring->stats.iocb_event++;

	/*
	 * The next available response entry should never exceed the maximum
	 * entries.  If it does, treat it as an adapter hardware error.
	 */
	portRspMax = pring->sli.sli3.numRiocb;
	portRspPut = le32_to_cpu(pgp->rspPutInx);
	if (unlikely(portRspPut >= portRspMax)) {
		lpfc_sli_rsp_pointers_error(phba, pring);
		spin_unlock_irqrestore(&phba->hbalock, iflag);
		return 1;
	}
	if (phba->fcp_ring_in_use) {
		spin_unlock_irqrestore(&phba->hbalock, iflag);
		return 1;
	} else
		phba->fcp_ring_in_use = 1;

	rmb();
	while (pring->sli.sli3.rspidx != portRspPut) {
		/*
		 * Fetch an entry off the ring and copy it into a local data
		 * structure.  The copy involves a byte-swap since the
		 * network byte order and pci byte orders are different.
		 */
		entry = lpfc_resp_iocb(phba, pring);
		phba->last_completion_time = jiffies;

		if (++pring->sli.sli3.rspidx >= portRspMax)
			pring->sli.sli3.rspidx = 0;

		lpfc_sli_pcimem_bcopy((uint32_t *) entry,
				      (uint32_t *) &rspiocbq.iocb,
				      phba->iocb_rsp_size);
		INIT_LIST_HEAD(&(rspiocbq.list));
		irsp = &rspiocbq.iocb;

		type = lpfc_sli_iocb_cmd_type(irsp->ulpCommand & CMD_IOCB_MASK);
		pring->stats.iocb_rsp++;
		rsp_cmpl++;

		if (unlikely(irsp->ulpStatus)) {
			/*
			 * If resource errors reported from HBA, reduce
			 * queuedepths of the SCSI device.
			 */
			if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
			    ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
			     IOERR_NO_RESOURCES)) {
				spin_unlock_irqrestore(&phba->hbalock, iflag);
				phba->lpfc_rampdown_queue_depth(phba);
				spin_lock_irqsave(&phba->hbalock, iflag);
			}

			/* Rsp ring <ringno> error: IOCB */
			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
					"0336 Rsp Ring %d error: IOCB Data: "
					"x%x x%x x%x x%x x%x x%x x%x x%x\n",
					pring->ringno,
					irsp->un.ulpWord[0],
					irsp->un.ulpWord[1],
					irsp->un.ulpWord[2],
					irsp->un.ulpWord[3],
					irsp->un.ulpWord[4],
					irsp->un.ulpWord[5],
					*(uint32_t *)&irsp->un1,
					*((uint32_t *)&irsp->un1 + 1));
		}

		switch (type) {
		case LPFC_ABORT_IOCB:
		case LPFC_SOL_IOCB:
			/*
			 * Idle exchange closed via ABTS from port.  No iocb
			 * resources need to be recovered.
			 */
			if (unlikely(irsp->ulpCommand == CMD_XRI_ABORTED_CX)) {
				lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
						"0333 IOCB cmd 0x%x"
						" processed. Skipping"
						" completion\n",
						irsp->ulpCommand);
				break;
			}

			spin_unlock_irqrestore(&phba->hbalock, iflag);
			cmdiocbq = lpfc_sli_iocbq_lookup(phba, pring,
							 &rspiocbq);
			spin_lock_irqsave(&phba->hbalock, iflag);
			if (unlikely(!cmdiocbq))
				break;
			if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED)
				cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED;
			if (cmdiocbq->iocb_cmpl) {
				spin_unlock_irqrestore(&phba->hbalock, iflag);
				(cmdiocbq->iocb_cmpl)(phba, cmdiocbq,
						      &rspiocbq);
				spin_lock_irqsave(&phba->hbalock, iflag);
			}
			break;
		case LPFC_UNSOL_IOCB:
			spin_unlock_irqrestore(&phba->hbalock, iflag);
			lpfc_sli_process_unsol_iocb(phba, pring, &rspiocbq);
			spin_lock_irqsave(&phba->hbalock, iflag);
			break;
		default:
			if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
				char adaptermsg[LPFC_MAX_ADPTMSG];
				memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
				memcpy(&adaptermsg[0], (uint8_t *) irsp,
				       MAX_MSG_DATA);
				dev_warn(&((phba->pcidev)->dev),
					 "lpfc%d: %s\n",
					 phba->brd_no, adaptermsg);
			} else {
				/* Unknown IOCB command */
				lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
						"0334 Unknown IOCB command "
						"Data: x%x, x%x x%x x%x x%x\n",
						type, irsp->ulpCommand,
						irsp->ulpStatus,
						irsp->ulpIoTag,
						irsp->ulpContext);
			}
			break;
		}

		/*
		 * The response IOCB has been processed.  Update the ring
		 * pointer in SLIM.  If the port response put pointer has not
		 * been updated, sync the pgp->rspPutInx and fetch the new port
		 * response put pointer.
		 */
		writel(pring->sli.sli3.rspidx,
			&phba->host_gp[pring->ringno].rspGetInx);

		if (pring->sli.sli3.rspidx == portRspPut)
			portRspPut = le32_to_cpu(pgp->rspPutInx);
	}

	if ((rsp_cmpl > 0) && (mask & HA_R0RE_REQ)) {
		pring->stats.iocb_rsp_full++;
		status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
		writel(status, phba->CAregaddr);
		readl(phba->CAregaddr);
	}
	if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
		pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
		pring->stats.iocb_cmd_empty++;

		/* Force update of the local copy of cmdGetInx */
		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
		lpfc_sli_resume_iocb(phba, pring);

		if ((pring->lpfc_sli_cmd_available))
			(pring->lpfc_sli_cmd_available) (phba, pring);

	}

	phba->fcp_ring_in_use = 0;
	spin_unlock_irqrestore(&phba->hbalock, iflag);
	return rc;
}

/**
 * lpfc_sli_sp_handle_rspiocb - Handle slow-path response iocb
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @rspiocbp: Pointer to driver response IOCB object.
 *
 * This function is called from the worker thread when there is a slow-path
 * response IOCB to process. This function chains all the response iocbs until
 * seeing the iocb with the LE bit set. The function will call
 * lpfc_sli_process_sol_iocb function if the response iocb indicates a
 * completion of a command iocb. The function will call the
 * lpfc_sli_process_unsol_iocb function if this is an unsolicited iocb.
 * The function frees the resources or calls the completion handler if this
 * iocb is an abort completion. The function returns NULL when the response
 * iocb has the LE bit set and all the chained iocbs are processed, otherwise
 * this function shall chain the iocb on to the iocb_continueq and return the
 * response iocb passed in.
 **/
static struct lpfc_iocbq *
lpfc_sli_sp_handle_rspiocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			struct lpfc_iocbq *rspiocbp)
{
	struct lpfc_iocbq *saveq;
	struct lpfc_iocbq *cmdiocbp;
	struct lpfc_iocbq *next_iocb;
	IOCB_t *irsp = NULL;
	uint32_t free_saveq;
	uint8_t iocb_cmd_type;
	lpfc_iocb_type type;
	unsigned long iflag;
	int rc;

	spin_lock_irqsave(&phba->hbalock, iflag);
	/* First add the response iocb to the countinueq list */
	list_add_tail(&rspiocbp->list, &(pring->iocb_continueq));
	pring->iocb_continueq_cnt++;

	/* Now, determine whether the list is completed for processing */
	irsp = &rspiocbp->iocb;
	if (irsp->ulpLe) {
		/*
		 * By default, the driver expects to free all resources
		 * associated with this iocb completion.
		 */
		free_saveq = 1;
		saveq = list_get_first(&pring->iocb_continueq,
				       struct lpfc_iocbq, list);
		irsp = &(saveq->iocb);
		list_del_init(&pring->iocb_continueq);
		pring->iocb_continueq_cnt = 0;

		pring->stats.iocb_rsp++;

		/*
		 * If resource errors reported from HBA, reduce
		 * queuedepths of the SCSI device.
		 */
		if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
		    ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
		     IOERR_NO_RESOURCES)) {
			spin_unlock_irqrestore(&phba->hbalock, iflag);
			phba->lpfc_rampdown_queue_depth(phba);
			spin_lock_irqsave(&phba->hbalock, iflag);
		}

		if (irsp->ulpStatus) {
			/* Rsp ring <ringno> error: IOCB */
			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
					"0328 Rsp Ring %d error: "
					"IOCB Data: "
					"x%x x%x x%x x%x "
					"x%x x%x x%x x%x "
					"x%x x%x x%x x%x "
					"x%x x%x x%x x%x\n",
					pring->ringno,
					irsp->un.ulpWord[0],
					irsp->un.ulpWord[1],
					irsp->un.ulpWord[2],
					irsp->un.ulpWord[3],
					irsp->un.ulpWord[4],
					irsp->un.ulpWord[5],
					*(((uint32_t *) irsp) + 6),
					*(((uint32_t *) irsp) + 7),
					*(((uint32_t *) irsp) + 8),
					*(((uint32_t *) irsp) + 9),
					*(((uint32_t *) irsp) + 10),
					*(((uint32_t *) irsp) + 11),
					*(((uint32_t *) irsp) + 12),
					*(((uint32_t *) irsp) + 13),
					*(((uint32_t *) irsp) + 14),
					*(((uint32_t *) irsp) + 15));
		}

		/*
		 * Fetch the IOCB command type and call the correct completion
		 * routine. Solicited and Unsolicited IOCBs on the ELS ring
		 * get freed back to the lpfc_iocb_list by the discovery
		 * kernel thread.
		 */
		iocb_cmd_type = irsp->ulpCommand & CMD_IOCB_MASK;
		type = lpfc_sli_iocb_cmd_type(iocb_cmd_type);
		switch (type) {
		case LPFC_SOL_IOCB:
			spin_unlock_irqrestore(&phba->hbalock, iflag);
			rc = lpfc_sli_process_sol_iocb(phba, pring, saveq);
			spin_lock_irqsave(&phba->hbalock, iflag);
			break;

		case LPFC_UNSOL_IOCB:
			spin_unlock_irqrestore(&phba->hbalock, iflag);
			rc = lpfc_sli_process_unsol_iocb(phba, pring, saveq);
			spin_lock_irqsave(&phba->hbalock, iflag);
			if (!rc)
				free_saveq = 0;
			break;

		case LPFC_ABORT_IOCB:
			cmdiocbp = NULL;
			if (irsp->ulpCommand != CMD_XRI_ABORTED_CX) {
				spin_unlock_irqrestore(&phba->hbalock, iflag);
				cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring,
								 saveq);
				spin_lock_irqsave(&phba->hbalock, iflag);
			}
			if (cmdiocbp) {
				/* Call the specified completion routine */
				if (cmdiocbp->iocb_cmpl) {
					spin_unlock_irqrestore(&phba->hbalock,
							       iflag);
					(cmdiocbp->iocb_cmpl)(phba, cmdiocbp,
							      saveq);
					spin_lock_irqsave(&phba->hbalock,
							  iflag);
				} else
					__lpfc_sli_release_iocbq(phba,
								 cmdiocbp);
			}
			break;

		case LPFC_UNKNOWN_IOCB:
			if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
				char adaptermsg[LPFC_MAX_ADPTMSG];
				memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
				memcpy(&adaptermsg[0], (uint8_t *)irsp,
				       MAX_MSG_DATA);
				dev_warn(&((phba->pcidev)->dev),
					 "lpfc%d: %s\n",
					 phba->brd_no, adaptermsg);
			} else {
				/* Unknown IOCB command */
				lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
						"0335 Unknown IOCB "
						"command Data: x%x "
						"x%x x%x x%x\n",
						irsp->ulpCommand,
						irsp->ulpStatus,
						irsp->ulpIoTag,
						irsp->ulpContext);
			}
			break;
		}

		if (free_saveq) {
			list_for_each_entry_safe(rspiocbp, next_iocb,
						 &saveq->list, list) {
				list_del_init(&rspiocbp->list);
				__lpfc_sli_release_iocbq(phba, rspiocbp);
			}
			__lpfc_sli_release_iocbq(phba, saveq);
		}
		rspiocbp = NULL;
	}
	spin_unlock_irqrestore(&phba->hbalock, iflag);
	return rspiocbp;
}

/**
 * lpfc_sli_handle_slow_ring_event - Wrapper func for handling slow-path iocbs
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mask: Host attention register mask for this ring.
 *
 * This routine wraps the actual slow_ring event process routine from the
 * API jump table function pointer from the lpfc_hba struct.
 **/
void
lpfc_sli_handle_slow_ring_event(struct lpfc_hba *phba,
				struct lpfc_sli_ring *pring, uint32_t mask)
{
	phba->lpfc_sli_handle_slow_ring_event(phba, pring, mask);
}

/**
 * lpfc_sli_handle_slow_ring_event_s3 - Handle SLI3 ring event for non-FCP rings
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mask: Host attention register mask for this ring.
 *
 * This function is called from the worker thread when there is a ring event
 * for non-fcp rings. The caller does not hold any lock. The function will
 * remove each response iocb in the response ring and calls the handle
 * response iocb routine (lpfc_sli_sp_handle_rspiocb) to process it.
 **/
static void
lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
				   struct lpfc_sli_ring *pring, uint32_t mask)
{
	struct lpfc_pgp *pgp;
	IOCB_t *entry;
	IOCB_t *irsp = NULL;
	struct lpfc_iocbq *rspiocbp = NULL;
	uint32_t portRspPut, portRspMax;
	unsigned long iflag;
	uint32_t status;

	pgp = &phba->port_gp[pring->ringno];
	spin_lock_irqsave(&phba->hbalock, iflag);
	pring->stats.iocb_event++;

	/*
	 * The next available response entry should never exceed the maximum
	 * entries.  If it does, treat it as an adapter hardware error.
	 */
	portRspMax = pring->sli.sli3.numRiocb;
	portRspPut = le32_to_cpu(pgp->rspPutInx);
	if (portRspPut >= portRspMax) {
		/*
		 * Ring <ringno> handler: portRspPut <portRspPut> is bigger than
		 * rsp ring <portRspMax>
		 */
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0303 Ring %d handler: portRspPut %d "
				"is bigger than rsp ring %d\n",
				pring->ringno, portRspPut, portRspMax);

		phba->link_state = LPFC_HBA_ERROR;
		spin_unlock_irqrestore(&phba->hbalock, iflag);

		phba->work_hs = HS_FFER3;
		lpfc_handle_eratt(phba);

		return;
	}

	rmb();
	while (pring->sli.sli3.rspidx != portRspPut) {
		/*
		 * Build a completion list and call the appropriate handler.
		 * The process is to get the next available response iocb, get
		 * a free iocb from the list, copy the response data into the
		 * free iocb, insert to the continuation list, and update the
		 * next response index to slim.  This process makes response
		 * iocb's in the ring available to DMA as fast as possible but
		 * pays a penalty for a copy operation.  Since the iocb is
		 * only 32 bytes, this penalty is considered small relative to
		 * the PCI reads for register values and a slim write.  When
		 * the ulpLe field is set, the entire Command has been
		 * received.
		 */
		entry = lpfc_resp_iocb(phba, pring);

		phba->last_completion_time = jiffies;
		rspiocbp = __lpfc_sli_get_iocbq(phba);
		if (rspiocbp == NULL) {
			printk(KERN_ERR "%s: out of buffers! Failing "
			       "completion.\n", __func__);
			break;
		}

		lpfc_sli_pcimem_bcopy(entry, &rspiocbp->iocb,
				      phba->iocb_rsp_size);
		irsp = &rspiocbp->iocb;

		if (++pring->sli.sli3.rspidx >= portRspMax)
			pring->sli.sli3.rspidx = 0;

		if (pring->ringno == LPFC_ELS_RING) {
			lpfc_debugfs_slow_ring_trc(phba,
			"IOCB rsp ring:   wd4:x%08x wd6:x%08x wd7:x%08x",
				*(((uint32_t *) irsp) + 4),
				*(((uint32_t *) irsp) + 6),
				*(((uint32_t *) irsp) + 7));
		}

		writel(pring->sli.sli3.rspidx,
			&phba->host_gp[pring->ringno].rspGetInx);

		spin_unlock_irqrestore(&phba->hbalock, iflag);
		/* Handle the response IOCB */
		rspiocbp = lpfc_sli_sp_handle_rspiocb(phba, pring, rspiocbp);
		spin_lock_irqsave(&phba->hbalock, iflag);

		/*
		 * If the port response put pointer has not been updated, sync
		 * the pgp->rspPutInx in the MAILBOX_tand fetch the new port
		 * response put pointer.
		 */
		if (pring->sli.sli3.rspidx == portRspPut) {
			portRspPut = le32_to_cpu(pgp->rspPutInx);
		}
	} /* while (pring->sli.sli3.rspidx != portRspPut) */

	if ((rspiocbp != NULL) && (mask & HA_R0RE_REQ)) {
		/* At least one response entry has been freed */
		pring->stats.iocb_rsp_full++;
		/* SET RxRE_RSP in Chip Att register */
		status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
		writel(status, phba->CAregaddr);
		readl(phba->CAregaddr); /* flush */
	}
	if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
		pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
		pring->stats.iocb_cmd_empty++;

		/* Force update of the local copy of cmdGetInx */
		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
		lpfc_sli_resume_iocb(phba, pring);

		if ((pring->lpfc_sli_cmd_available))
			(pring->lpfc_sli_cmd_available) (phba, pring);

	}

	spin_unlock_irqrestore(&phba->hbalock, iflag);
	return;
}

/**
 * lpfc_sli_handle_slow_ring_event_s4 - Handle SLI4 slow-path els events
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mask: Host attention register mask for this ring.
 *
 * This function is called from the worker thread when there is a pending
 * ELS response iocb on the driver internal slow-path response iocb worker
 * queue. The caller does not hold any lock. The function will remove each
 * response iocb from the response worker queue and calls the handle
 * response iocb routine (lpfc_sli_sp_handle_rspiocb) to process it.
 **/
static void
lpfc_sli_handle_slow_ring_event_s4(struct lpfc_hba *phba,
				   struct lpfc_sli_ring *pring, uint32_t mask)
{
	struct lpfc_iocbq *irspiocbq;
	struct hbq_dmabuf *dmabuf;
	struct lpfc_cq_event *cq_event;
	unsigned long iflag;
	int count = 0;

	spin_lock_irqsave(&phba->hbalock, iflag);
	phba->hba_flag &= ~HBA_SP_QUEUE_EVT;
	spin_unlock_irqrestore(&phba->hbalock, iflag);
	while (!list_empty(&phba->sli4_hba.sp_queue_event)) {
		/* Get the response iocb from the head of work queue */
		spin_lock_irqsave(&phba->hbalock, iflag);
		list_remove_head(&phba->sli4_hba.sp_queue_event,
				 cq_event, struct lpfc_cq_event, list);
		spin_unlock_irqrestore(&phba->hbalock, iflag);

		switch (bf_get(lpfc_wcqe_c_code, &cq_event->cqe.wcqe_cmpl)) {
		case CQE_CODE_COMPL_WQE:
			irspiocbq = container_of(cq_event, struct lpfc_iocbq,
						 cq_event);
			/* Translate ELS WCQE to response IOCBQ */
			irspiocbq = lpfc_sli4_els_wcqe_to_rspiocbq(phba,
								   irspiocbq);
			if (irspiocbq)
				lpfc_sli_sp_handle_rspiocb(phba, pring,
							   irspiocbq);
			count++;
			break;
		case CQE_CODE_RECEIVE:
		case CQE_CODE_RECEIVE_V1:
			dmabuf = container_of(cq_event, struct hbq_dmabuf,
					      cq_event);
			lpfc_sli4_handle_received_buffer(phba, dmabuf);
			count++;
			break;
		default:
			break;
		}

		/* Limit the number of events to 64 to avoid soft lockups */
		if (count == 64)
			break;
	}
}

/**
 * lpfc_sli_abort_iocb_ring - Abort all iocbs in the ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function aborts all iocbs in the given ring and frees all the iocb
 * objects in txq. This function issues an abort iocb for all the iocb commands
 * in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
 * the return of this function. The caller is not required to hold any locks.
 **/
void
lpfc_sli_abort_iocb_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
	LIST_HEAD(completions);
	struct lpfc_iocbq *iocb, *next_iocb;

	if (pring->ringno == LPFC_ELS_RING) {
		lpfc_fabric_abort_hba(phba);
	}

	/* Error everything on txq and txcmplq
	 * First do the txq.
	 */
	if (phba->sli_rev >= LPFC_SLI_REV4) {
		spin_lock_irq(&pring->ring_lock);
		list_splice_init(&pring->txq, &completions);
		pring->txq_cnt = 0;
		spin_unlock_irq(&pring->ring_lock);

		spin_lock_irq(&phba->hbalock);
		/* Next issue ABTS for everything on the txcmplq */
		list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
			lpfc_sli_issue_abort_iotag(phba, pring, iocb);
		spin_unlock_irq(&phba->hbalock);
	} else {
		spin_lock_irq(&phba->hbalock);
		list_splice_init(&pring->txq, &completions);
		pring->txq_cnt = 0;

		/* Next issue ABTS for everything on the txcmplq */
		list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
			lpfc_sli_issue_abort_iotag(phba, pring, iocb);
		spin_unlock_irq(&phba->hbalock);
	}

	/* Cancel all the IOCBs from the completions list */
	lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
			      IOERR_SLI_ABORTED);
}

/**
 * lpfc_sli_abort_fcp_rings - Abort all iocbs in all FCP rings
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function aborts all iocbs in FCP rings and frees all the iocb
 * objects in txq. This function issues an abort iocb for all the iocb commands
 * in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
 * the return of this function. The caller is not required to hold any locks.
 **/
void
lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;
	struct lpfc_sli_ring  *pring;
	uint32_t i;

	/* Look on all the FCP Rings for the iotag */
	if (phba->sli_rev >= LPFC_SLI_REV4) {
		for (i = 0; i < phba->cfg_hdw_queue; i++) {
			pring = phba->sli4_hba.hdwq[i].fcp_wq->pring;
			lpfc_sli_abort_iocb_ring(phba, pring);
		}
	} else {
		pring = &psli->sli3_ring[LPFC_FCP_RING];
		lpfc_sli_abort_iocb_ring(phba, pring);
	}
}

/**
 * lpfc_sli_flush_fcp_rings - flush all iocbs in the fcp ring
 * @phba: Pointer to HBA context object.
 *
 * This function flushes all iocbs in the fcp ring and frees all the iocb
 * objects in txq and txcmplq. This function will not issue abort iocbs
 * for all the iocb commands in txcmplq, they will just be returned with
 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
 * slot has been permanently disabled.
 **/
void
lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba)
{
	LIST_HEAD(txq);
	LIST_HEAD(txcmplq);
	struct lpfc_sli *psli = &phba->sli;
	struct lpfc_sli_ring  *pring;
	uint32_t i;
	struct lpfc_iocbq *piocb, *next_iocb;

	spin_lock_irq(&phba->hbalock);
	/* Indicate the I/O queues are flushed */
	phba->hba_flag |= HBA_FCP_IOQ_FLUSH;
	spin_unlock_irq(&phba->hbalock);

	/* Look on all the FCP Rings for the iotag */
	if (phba->sli_rev >= LPFC_SLI_REV4) {
		for (i = 0; i < phba->cfg_hdw_queue; i++) {
			pring = phba->sli4_hba.hdwq[i].fcp_wq->pring;

			spin_lock_irq(&pring->ring_lock);
			/* Retrieve everything on txq */
			list_splice_init(&pring->txq, &txq);
			list_for_each_entry_safe(piocb, next_iocb,
						 &pring->txcmplq, list)
				piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
			/* Retrieve everything on the txcmplq */
			list_splice_init(&pring->txcmplq, &txcmplq);
			pring->txq_cnt = 0;
			pring->txcmplq_cnt = 0;
			spin_unlock_irq(&pring->ring_lock);

			/* Flush the txq */
			lpfc_sli_cancel_iocbs(phba, &txq,
					      IOSTAT_LOCAL_REJECT,
					      IOERR_SLI_DOWN);
			/* Flush the txcmpq */
			lpfc_sli_cancel_iocbs(phba, &txcmplq,
					      IOSTAT_LOCAL_REJECT,
					      IOERR_SLI_DOWN);
		}
	} else {
		pring = &psli->sli3_ring[LPFC_FCP_RING];

		spin_lock_irq(&phba->hbalock);
		/* Retrieve everything on txq */
		list_splice_init(&pring->txq, &txq);
		list_for_each_entry_safe(piocb, next_iocb,
					 &pring->txcmplq, list)
			piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
		/* Retrieve everything on the txcmplq */
		list_splice_init(&pring->txcmplq, &txcmplq);
		pring->txq_cnt = 0;
		pring->txcmplq_cnt = 0;
		spin_unlock_irq(&phba->hbalock);

		/* Flush the txq */
		lpfc_sli_cancel_iocbs(phba, &txq, IOSTAT_LOCAL_REJECT,
				      IOERR_SLI_DOWN);
		/* Flush the txcmpq */
		lpfc_sli_cancel_iocbs(phba, &txcmplq, IOSTAT_LOCAL_REJECT,
				      IOERR_SLI_DOWN);
	}
}

/**
 * lpfc_sli_flush_nvme_rings - flush all wqes in the nvme rings
 * @phba: Pointer to HBA context object.
 *
 * This function flushes all wqes in the nvme rings and frees all resources
 * in the txcmplq. This function does not issue abort wqes for the IO
 * commands in txcmplq, they will just be returned with
 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
 * slot has been permanently disabled.
 **/
void
lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba)
{
	LIST_HEAD(txcmplq);
	struct lpfc_sli_ring  *pring;
	uint32_t i;
	struct lpfc_iocbq *piocb, *next_iocb;

	if ((phba->sli_rev < LPFC_SLI_REV4) ||
	    !(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
		return;

	/* Hint to other driver operations that a flush is in progress. */
	spin_lock_irq(&phba->hbalock);
	phba->hba_flag |= HBA_NVME_IOQ_FLUSH;
	spin_unlock_irq(&phba->hbalock);

	/* Cycle through all NVME rings and complete each IO with
	 * a local driver reason code.  This is a flush so no
	 * abort exchange to FW.
	 */
	for (i = 0; i < phba->cfg_hdw_queue; i++) {
		pring = phba->sli4_hba.hdwq[i].nvme_wq->pring;

		spin_lock_irq(&pring->ring_lock);
		list_for_each_entry_safe(piocb, next_iocb,
					 &pring->txcmplq, list)
			piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
		/* Retrieve everything on the txcmplq */
		list_splice_init(&pring->txcmplq, &txcmplq);
		pring->txcmplq_cnt = 0;
		spin_unlock_irq(&pring->ring_lock);

		/* Flush the txcmpq &&&PAE */
		lpfc_sli_cancel_iocbs(phba, &txcmplq,
				      IOSTAT_LOCAL_REJECT,
				      IOERR_SLI_DOWN);
	}
}

/**
 * lpfc_sli_brdready_s3 - Check for sli3 host ready status
 * @phba: Pointer to HBA context object.
 * @mask: Bit mask to be checked.
 *
 * This function reads the host status register and compares
 * with the provided bit mask to check if HBA completed
 * the restart. This function will wait in a loop for the
 * HBA to complete restart. If the HBA does not restart within
 * 15 iterations, the function will reset the HBA again. The
 * function returns 1 when HBA fail to restart otherwise returns
 * zero.
 **/
static int
lpfc_sli_brdready_s3(struct lpfc_hba *phba, uint32_t mask)
{
	uint32_t status;
	int i = 0;
	int retval = 0;

	/* Read the HBA Host Status Register */
	if (lpfc_readl(phba->HSregaddr, &status))
		return 1;

	/*
	 * Check status register every 100ms for 5 retries, then every
	 * 500ms for 5, then every 2.5 sec for 5, then reset board and
	 * every 2.5 sec for 4.
	 * Break our of the loop if errors occurred during init.
	 */
	while (((status & mask) != mask) &&
	       !(status & HS_FFERM) &&
	       i++ < 20) {

		if (i <= 5)
			msleep(10);
		else if (i <= 10)
			msleep(500);
		else
			msleep(2500);

		if (i == 15) {
				/* Do post */
			phba->pport->port_state = LPFC_VPORT_UNKNOWN;
			lpfc_sli_brdrestart(phba);
		}
		/* Read the HBA Host Status Register */
		if (lpfc_readl(phba->HSregaddr, &status)) {
			retval = 1;
			break;
		}
	}

	/* Check to see if any errors occurred during init */
	if ((status & HS_FFERM) || (i >= 20)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2751 Adapter failed to restart, "
				"status reg x%x, FW Data: A8 x%x AC x%x\n",
				status,
				readl(phba->MBslimaddr + 0xa8),
				readl(phba->MBslimaddr + 0xac));
		phba->link_state = LPFC_HBA_ERROR;
		retval = 1;
	}

	return retval;
}

/**
 * lpfc_sli_brdready_s4 - Check for sli4 host ready status
 * @phba: Pointer to HBA context object.
 * @mask: Bit mask to be checked.
 *
 * This function checks the host status register to check if HBA is
 * ready. This function will wait in a loop for the HBA to be ready
 * If the HBA is not ready , the function will will reset the HBA PCI
 * function again. The function returns 1 when HBA fail to be ready
 * otherwise returns zero.
 **/
static int
lpfc_sli_brdready_s4(struct lpfc_hba *phba, uint32_t mask)
{
	uint32_t status;
	int retval = 0;

	/* Read the HBA Host Status Register */
	status = lpfc_sli4_post_status_check(phba);

	if (status) {
		phba->pport->port_state = LPFC_VPORT_UNKNOWN;
		lpfc_sli_brdrestart(phba);
		status = lpfc_sli4_post_status_check(phba);
	}

	/* Check to see if any errors occurred during init */
	if (status) {
		phba->link_state = LPFC_HBA_ERROR;
		retval = 1;
	} else
		phba->sli4_hba.intr_enable = 0;

	return retval;
}

/**
 * lpfc_sli_brdready - Wrapper func for checking the hba readyness
 * @phba: Pointer to HBA context object.
 * @mask: Bit mask to be checked.
 *
 * This routine wraps the actual SLI3 or SLI4 hba readyness check routine
 * from the API jump table function pointer from the lpfc_hba struct.
 **/
int
lpfc_sli_brdready(struct lpfc_hba *phba, uint32_t mask)
{
	return phba->lpfc_sli_brdready(phba, mask);
}

#define BARRIER_TEST_PATTERN (0xdeadbeef)

/**
 * lpfc_reset_barrier - Make HBA ready for HBA reset
 * @phba: Pointer to HBA context object.
 *
 * This function is called before resetting an HBA. This function is called
 * with hbalock held and requests HBA to quiesce DMAs before a reset.
 **/
void lpfc_reset_barrier(struct lpfc_hba *phba)
{
	uint32_t __iomem *resp_buf;
	uint32_t __iomem *mbox_buf;
	volatile uint32_t mbox;
	uint32_t hc_copy, ha_copy, resp_data;
	int  i;
	uint8_t hdrtype;

	lockdep_assert_held(&phba->hbalock);

	pci_read_config_byte(phba->pcidev, PCI_HEADER_TYPE, &hdrtype);
	if (hdrtype != 0x80 ||
	    (FC_JEDEC_ID(phba->vpd.rev.biuRev) != HELIOS_JEDEC_ID &&
	     FC_JEDEC_ID(phba->vpd.rev.biuRev) != THOR_JEDEC_ID))
		return;

	/*
	 * Tell the other part of the chip to suspend temporarily all
	 * its DMA activity.
	 */
	resp_buf = phba->MBslimaddr;

	/* Disable the error attention */
	if (lpfc_readl(phba->HCregaddr, &hc_copy))
		return;
	writel((hc_copy & ~HC_ERINT_ENA), phba->HCregaddr);
	readl(phba->HCregaddr); /* flush */
	phba->link_flag |= LS_IGNORE_ERATT;

	if (lpfc_readl(phba->HAregaddr, &ha_copy))
		return;
	if (ha_copy & HA_ERATT) {
		/* Clear Chip error bit */
		writel(HA_ERATT, phba->HAregaddr);
		phba->pport->stopped = 1;
	}

	mbox = 0;
	((MAILBOX_t *)&mbox)->mbxCommand = MBX_KILL_BOARD;
	((MAILBOX_t *)&mbox)->mbxOwner = OWN_CHIP;

	writel(BARRIER_TEST_PATTERN, (resp_buf + 1));
	mbox_buf = phba->MBslimaddr;
	writel(mbox, mbox_buf);

	for (i = 0; i < 50; i++) {
		if (lpfc_readl((resp_buf + 1), &resp_data))
			return;
		if (resp_data != ~(BARRIER_TEST_PATTERN))
			mdelay(1);
		else
			break;
	}
	resp_data = 0;
	if (lpfc_readl((resp_buf + 1), &resp_data))
		return;
	if (resp_data  != ~(BARRIER_TEST_PATTERN)) {
		if (phba->sli.sli_flag & LPFC_SLI_ACTIVE ||
		    phba->pport->stopped)
			goto restore_hc;
		else
			goto clear_errat;
	}

	((MAILBOX_t *)&mbox)->mbxOwner = OWN_HOST;
	resp_data = 0;
	for (i = 0; i < 500; i++) {
		if (lpfc_readl(resp_buf, &resp_data))
			return;
		if (resp_data != mbox)
			mdelay(1);
		else
			break;
	}

clear_errat:

	while (++i < 500) {
		if (lpfc_readl(phba->HAregaddr, &ha_copy))
			return;
		if (!(ha_copy & HA_ERATT))
			mdelay(1);
		else
			break;
	}

	if (readl(phba->HAregaddr) & HA_ERATT) {
		writel(HA_ERATT, phba->HAregaddr);
		phba->pport->stopped = 1;
	}

restore_hc:
	phba->link_flag &= ~LS_IGNORE_ERATT;
	writel(hc_copy, phba->HCregaddr);
	readl(phba->HCregaddr); /* flush */
}

/**
 * lpfc_sli_brdkill - Issue a kill_board mailbox command
 * @phba: Pointer to HBA context object.
 *
 * This function issues a kill_board mailbox command and waits for
 * the error attention interrupt. This function is called for stopping
 * the firmware processing. The caller is not required to hold any
 * locks. This function calls lpfc_hba_down_post function to free
 * any pending commands after the kill. The function will return 1 when it
 * fails to kill the board else will return 0.
 **/
int
lpfc_sli_brdkill(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli;
	LPFC_MBOXQ_t *pmb;
	uint32_t status;
	uint32_t ha_copy;
	int retval;
	int i = 0;

	psli = &phba->sli;

	/* Kill HBA */
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"0329 Kill HBA Data: x%x x%x\n",
			phba->pport->port_state, psli->sli_flag);

	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!pmb)
		return 1;

	/* Disable the error attention */
	spin_lock_irq(&phba->hbalock);
	if (lpfc_readl(phba->HCregaddr, &status)) {
		spin_unlock_irq(&phba->hbalock);
		mempool_free(pmb, phba->mbox_mem_pool);
		return 1;
	}
	status &= ~HC_ERINT_ENA;
	writel(status, phba->HCregaddr);
	readl(phba->HCregaddr); /* flush */
	phba->link_flag |= LS_IGNORE_ERATT;
	spin_unlock_irq(&phba->hbalock);

	lpfc_kill_board(phba, pmb);
	pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
	retval = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);

	if (retval != MBX_SUCCESS) {
		if (retval != MBX_BUSY)
			mempool_free(pmb, phba->mbox_mem_pool);
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"2752 KILL_BOARD command failed retval %d\n",
				retval);
		spin_lock_irq(&phba->hbalock);
		phba->link_flag &= ~LS_IGNORE_ERATT;
		spin_unlock_irq(&phba->hbalock);
		return 1;
	}

	spin_lock_irq(&phba->hbalock);
	psli->sli_flag &= ~LPFC_SLI_ACTIVE;
	spin_unlock_irq(&phba->hbalock);

	mempool_free(pmb, phba->mbox_mem_pool);

	/* There is no completion for a KILL_BOARD mbox cmd. Check for an error
	 * attention every 100ms for 3 seconds. If we don't get ERATT after
	 * 3 seconds we still set HBA_ERROR state because the status of the
	 * board is now undefined.
	 */
	if (lpfc_readl(phba->HAregaddr, &ha_copy))
		return 1;
	while ((i++ < 30) && !(ha_copy & HA_ERATT)) {
		mdelay(100);
		if (lpfc_readl(phba->HAregaddr, &ha_copy))
			return 1;
	}

	del_timer_sync(&psli->mbox_tmo);
	if (ha_copy & HA_ERATT) {
		writel(HA_ERATT, phba->HAregaddr);
		phba->pport->stopped = 1;
	}
	spin_lock_irq(&phba->hbalock);
	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
	psli->mbox_active = NULL;
	phba->link_flag &= ~LS_IGNORE_ERATT;
	spin_unlock_irq(&phba->hbalock);

	lpfc_hba_down_post(phba);
	phba->link_state = LPFC_HBA_ERROR;

	return ha_copy & HA_ERATT ? 0 : 1;
}

/**
 * lpfc_sli_brdreset - Reset a sli-2 or sli-3 HBA
 * @phba: Pointer to HBA context object.
 *
 * This function resets the HBA by writing HC_INITFF to the control
 * register. After the HBA resets, this function resets all the iocb ring
 * indices. This function disables PCI layer parity checking during
 * the reset.
 * This function returns 0 always.
 * The caller is not required to hold any locks.
 **/
int
lpfc_sli_brdreset(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli;
	struct lpfc_sli_ring *pring;
	uint16_t cfg_value;
	int i;

	psli = &phba->sli;

	/* Reset HBA */
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"0325 Reset HBA Data: x%x x%x\n",
			(phba->pport) ? phba->pport->port_state : 0,
			psli->sli_flag);

	/* perform board reset */
	phba->fc_eventTag = 0;
	phba->link_events = 0;
	if (phba->pport) {
		phba->pport->fc_myDID = 0;
		phba->pport->fc_prevDID = 0;
	}

	/* Turn off parity checking and serr during the physical reset */
	if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value))
		return -EIO;

	pci_write_config_word(phba->pcidev, PCI_COMMAND,
			      (cfg_value &
			       ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));

	psli->sli_flag &= ~(LPFC_SLI_ACTIVE | LPFC_PROCESS_LA);

	/* Now toggle INITFF bit in the Host Control Register */
	writel(HC_INITFF, phba->HCregaddr);
	mdelay(1);
	readl(phba->HCregaddr); /* flush */
	writel(0, phba->HCregaddr);
	readl(phba->HCregaddr); /* flush */

	/* Restore PCI cmd register */
	pci_write_config_word(phba->pcidev, PCI_COMMAND, cfg_value);

	/* Initialize relevant SLI info */
	for (i = 0; i < psli->num_rings; i++) {
		pring = &psli->sli3_ring[i];
		pring->flag = 0;
		pring->sli.sli3.rspidx = 0;
		pring->sli.sli3.next_cmdidx  = 0;
		pring->sli.sli3.local_getidx = 0;
		pring->sli.sli3.cmdidx = 0;
		pring->missbufcnt = 0;
	}

	phba->link_state = LPFC_WARM_START;
	return 0;
}

/**
 * lpfc_sli4_brdreset - Reset a sli-4 HBA
 * @phba: Pointer to HBA context object.
 *
 * This function resets a SLI4 HBA. This function disables PCI layer parity
 * checking during resets the device. The caller is not required to hold
 * any locks.
 *
 * This function returns 0 always.
 **/
int
lpfc_sli4_brdreset(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;
	uint16_t cfg_value;
	int rc = 0;

	/* Reset HBA */
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"0295 Reset HBA Data: x%x x%x x%x\n",
			phba->pport->port_state, psli->sli_flag,
			phba->hba_flag);

	/* perform board reset */
	phba->fc_eventTag = 0;
	phba->link_events = 0;
	phba->pport->fc_myDID = 0;
	phba->pport->fc_prevDID = 0;

	spin_lock_irq(&phba->hbalock);
	psli->sli_flag &= ~(LPFC_PROCESS_LA);
	phba->fcf.fcf_flag = 0;
	spin_unlock_irq(&phba->hbalock);

	/* SLI4 INTF 2: if FW dump is being taken skip INIT_PORT */
	if (phba->hba_flag & HBA_FW_DUMP_OP) {
		phba->hba_flag &= ~HBA_FW_DUMP_OP;
		return rc;
	}

	/* Now physically reset the device */
	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
			"0389 Performing PCI function reset!\n");

	/* Turn off parity checking and serr during the physical reset */
	if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value)) {
		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
				"3205 PCI read Config failed\n");
		return -EIO;
	}

	pci_write_config_word(phba->pcidev, PCI_COMMAND, (cfg_value &
			      ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));

	/* Perform FCoE PCI function reset before freeing queue memory */
	rc = lpfc_pci_function_reset(phba);

	/* Restore PCI cmd register */
	pci_write_config_word(phba->pcidev, PCI_COMMAND, cfg_value);

	return rc;
}

/**
 * lpfc_sli_brdrestart_s3 - Restart a sli-3 hba
 * @phba: Pointer to HBA context object.
 *
 * This function is called in the SLI initialization code path to
 * restart the HBA. The caller is not required to hold any lock.
 * This function writes MBX_RESTART mailbox command to the SLIM and
 * resets the HBA. At the end of the function, it calls lpfc_hba_down_post
 * function to free any pending commands. The function enables
 * POST only during the first initialization. The function returns zero.
 * The function does not guarantee completion of MBX_RESTART mailbox
 * command before the return of this function.
 **/
static int
lpfc_sli_brdrestart_s3(struct lpfc_hba *phba)
{
	MAILBOX_t *mb;
	struct lpfc_sli *psli;
	volatile uint32_t word0;
	void __iomem *to_slim;
	uint32_t hba_aer_enabled;

	spin_lock_irq(&phba->hbalock);

	/* Take PCIe device Advanced Error Reporting (AER) state */
	hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED;

	psli = &phba->sli;

	/* Restart HBA */
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"0337 Restart HBA Data: x%x x%x\n",
			(phba->pport) ? phba->pport->port_state : 0,
			psli->sli_flag);

	word0 = 0;
	mb = (MAILBOX_t *) &word0;
	mb->mbxCommand = MBX_RESTART;
	mb->mbxHc = 1;

	lpfc_reset_barrier(phba);

	to_slim = phba->MBslimaddr;
	writel(*(uint32_t *) mb, to_slim);
	readl(to_slim); /* flush */

	/* Only skip post after fc_ffinit is completed */
	if (phba->pport && phba->pport->port_state)
		word0 = 1;	/* This is really setting up word1 */
	else
		word0 = 0;	/* This is really setting up word1 */
	to_slim = phba->MBslimaddr + sizeof (uint32_t);
	writel(*(uint32_t *) mb, to_slim);
	readl(to_slim); /* flush */

	lpfc_sli_brdreset(phba);
	if (phba->pport)
		phba->pport->stopped = 0;
	phba->link_state = LPFC_INIT_START;
	phba->hba_flag = 0;
	spin_unlock_irq(&phba->hbalock);

	memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets));
	psli->stats_start = ktime_get_seconds();

	/* Give the INITFF and Post time to settle. */
	mdelay(100);

	/* Reset HBA AER if it was enabled, note hba_flag was reset above */
	if (hba_aer_enabled)
		pci_disable_pcie_error_reporting(phba->pcidev);

	lpfc_hba_down_post(phba);

	return 0;
}

/**
 * lpfc_sli_brdrestart_s4 - Restart the sli-4 hba
 * @phba: Pointer to HBA context object.
 *
 * This function is called in the SLI initialization code path to restart
 * a SLI4 HBA. The caller is not required to hold any lock.
 * At the end of the function, it calls lpfc_hba_down_post function to
 * free any pending commands.
 **/
static int
lpfc_sli_brdrestart_s4(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;
	uint32_t hba_aer_enabled;
	int rc;

	/* Restart HBA */
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"0296 Restart HBA Data: x%x x%x\n",
			phba->pport->port_state, psli->sli_flag);

	/* Take PCIe device Advanced Error Reporting (AER) state */
	hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED;

	rc = lpfc_sli4_brdreset(phba);
	if (rc)
		return rc;

	spin_lock_irq(&phba->hbalock);
	phba->pport->stopped = 0;
	phba->link_state = LPFC_INIT_START;
	phba->hba_flag = 0;
	spin_unlock_irq(&phba->hbalock);

	memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets));
	psli->stats_start = ktime_get_seconds();

	/* Reset HBA AER if it was enabled, note hba_flag was reset above */
	if (hba_aer_enabled)
		pci_disable_pcie_error_reporting(phba->pcidev);

	lpfc_hba_down_post(phba);
	lpfc_sli4_queue_destroy(phba);

	return rc;
}

/**
 * lpfc_sli_brdrestart - Wrapper func for restarting hba
 * @phba: Pointer to HBA context object.
 *
 * This routine wraps the actual SLI3 or SLI4 hba restart routine from the
 * API jump table function pointer from the lpfc_hba struct.
**/
int
lpfc_sli_brdrestart(struct lpfc_hba *phba)
{
	return phba->lpfc_sli_brdrestart(phba);
}

/**
 * lpfc_sli_chipset_init - Wait for the restart of the HBA after a restart
 * @phba: Pointer to HBA context object.
 *
 * This function is called after a HBA restart to wait for successful
 * restart of the HBA. Successful restart of the HBA is indicated by
 * HS_FFRDY and HS_MBRDY bits. If the HBA fails to restart even after 15
 * iteration, the function will restart the HBA again. The function returns
 * zero if HBA successfully restarted else returns negative error code.
 **/
int
lpfc_sli_chipset_init(struct lpfc_hba *phba)
{
	uint32_t status, i = 0;

	/* Read the HBA Host Status Register */
	if (lpfc_readl(phba->HSregaddr, &status))
		return -EIO;

	/* Check status register to see what current state is */
	i = 0;
	while ((status & (HS_FFRDY | HS_MBRDY)) != (HS_FFRDY | HS_MBRDY)) {

		/* Check every 10ms for 10 retries, then every 100ms for 90
		 * retries, then every 1 sec for 50 retires for a total of
		 * ~60 seconds before reset the board again and check every
		 * 1 sec for 50 retries. The up to 60 seconds before the
		 * board ready is required by the Falcon FIPS zeroization
		 * complete, and any reset the board in between shall cause
		 * restart of zeroization, further delay the board ready.
		 */
		if (i++ >= 200) {
			/* Adapter failed to init, timeout, status reg
			   <status> */
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"0436 Adapter failed to init, "
					"timeout, status reg x%x, "
					"FW Data: A8 x%x AC x%x\n", status,
					readl(phba->MBslimaddr + 0xa8),
					readl(phba->MBslimaddr + 0xac));
			phba->link_state = LPFC_HBA_ERROR;
			return -ETIMEDOUT;
		}

		/* Check to see if any errors occurred during init */
		if (status & HS_FFERM) {
			/* ERROR: During chipset initialization */
			/* Adapter failed to init, chipset, status reg
			   <status> */
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"0437 Adapter failed to init, "
					"chipset, status reg x%x, "
					"FW Data: A8 x%x AC x%x\n", status,
					readl(phba->MBslimaddr + 0xa8),
					readl(phba->MBslimaddr + 0xac));
			phba->link_state = LPFC_HBA_ERROR;
			return -EIO;
		}

		if (i <= 10)
			msleep(10);
		else if (i <= 100)
			msleep(100);
		else
			msleep(1000);

		if (i == 150) {
			/* Do post */
			phba->pport->port_state = LPFC_VPORT_UNKNOWN;
			lpfc_sli_brdrestart(phba);
		}
		/* Read the HBA Host Status Register */
		if (lpfc_readl(phba->HSregaddr, &status))
			return -EIO;
	}

	/* Check to see if any errors occurred during init */
	if (status & HS_FFERM) {
		/* ERROR: During chipset initialization */
		/* Adapter failed to init, chipset, status reg <status> */
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"0438 Adapter failed to init, chipset, "
				"status reg x%x, "
				"FW Data: A8 x%x AC x%x\n", status,
				readl(phba->MBslimaddr + 0xa8),
				readl(phba->MBslimaddr + 0xac));
		phba->link_state = LPFC_HBA_ERROR;
		return -EIO;
	}

	/* Clear all interrupt enable conditions */
	writel(0, phba->HCregaddr);
	readl(phba->HCregaddr); /* flush */

	/* setup host attn register */
	writel(0xffffffff, phba->HAregaddr);
	readl(phba->HAregaddr); /* flush */
	return 0;
}

/**
 * lpfc_sli_hbq_count - Get the number of HBQs to be configured
 *
 * This function calculates and returns the number of HBQs required to be
 * configured.
 **/
int
lpfc_sli_hbq_count(void)
{
	return ARRAY_SIZE(lpfc_hbq_defs);
}

/**
 * lpfc_sli_hbq_entry_count - Calculate total number of hbq entries
 *
 * This function adds the number of hbq entries in every HBQ to get
 * the total number of hbq entries required for the HBA and returns
 * the total count.
 **/
static int
lpfc_sli_hbq_entry_count(void)
{
	int  hbq_count = lpfc_sli_hbq_count();
	int  count = 0;
	int  i;

	for (i = 0; i < hbq_count; ++i)
		count += lpfc_hbq_defs[i]->entry_count;
	return count;
}

/**
 * lpfc_sli_hbq_size - Calculate memory required for all hbq entries
 *
 * This function calculates amount of memory required for all hbq entries
 * to be configured and returns the total memory required.
 **/
int
lpfc_sli_hbq_size(void)
{
	return lpfc_sli_hbq_entry_count() * sizeof(struct lpfc_hbq_entry);
}

/**
 * lpfc_sli_hbq_setup - configure and initialize HBQs
 * @phba: Pointer to HBA context object.
 *
 * This function is called during the SLI initialization to configure
 * all the HBQs and post buffers to the HBQ. The caller is not
 * required to hold any locks. This function will return zero if successful
 * else it will return negative error code.
 **/
static int
lpfc_sli_hbq_setup(struct lpfc_hba *phba)
{
	int  hbq_count = lpfc_sli_hbq_count();
	LPFC_MBOXQ_t *pmb;
	MAILBOX_t *pmbox;
	uint32_t hbqno;
	uint32_t hbq_entry_index;

				/* Get a Mailbox buffer to setup mailbox
				 * commands for HBA initialization
				 */
	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);

	if (!pmb)
		return -ENOMEM;

	pmbox = &pmb->u.mb;

	/* Initialize the struct lpfc_sli_hbq structure for each hbq */
	phba->link_state = LPFC_INIT_MBX_CMDS;
	phba->hbq_in_use = 1;

	hbq_entry_index = 0;
	for (hbqno = 0; hbqno < hbq_count; ++hbqno) {
		phba->hbqs[hbqno].next_hbqPutIdx = 0;
		phba->hbqs[hbqno].hbqPutIdx      = 0;
		phba->hbqs[hbqno].local_hbqGetIdx   = 0;
		phba->hbqs[hbqno].entry_count =
			lpfc_hbq_defs[hbqno]->entry_count;
		lpfc_config_hbq(phba, hbqno, lpfc_hbq_defs[hbqno],
			hbq_entry_index, pmb);
		hbq_entry_index += phba->hbqs[hbqno].entry_count;

		if (lpfc_sli_issue_mbox(phba, pmb, MBX_POLL) != MBX_SUCCESS) {
			/* Adapter failed to init, mbxCmd <cmd> CFG_RING,
			   mbxStatus <status>, ring <num> */

			lpfc_printf_log(phba, KERN_ERR,
					LOG_SLI | LOG_VPORT,
					"1805 Adapter failed to init. "
					"Data: x%x x%x x%x\n",
					pmbox->mbxCommand,
					pmbox->mbxStatus, hbqno);

			phba->link_state = LPFC_HBA_ERROR;
			mempool_free(pmb, phba->mbox_mem_pool);
			return -ENXIO;
		}
	}
	phba->hbq_count = hbq_count;

	mempool_free(pmb, phba->mbox_mem_pool);

	/* Initially populate or replenish the HBQs */
	for (hbqno = 0; hbqno < hbq_count; ++hbqno)
		lpfc_sli_hbqbuf_init_hbqs(phba, hbqno);
	return 0;
}

/**
 * lpfc_sli4_rb_setup - Initialize and post RBs to HBA
 * @phba: Pointer to HBA context object.
 *
 * This function is called during the SLI initialization to configure
 * all the HBQs and post buffers to the HBQ. The caller is not
 * required to hold any locks. This function will return zero if successful
 * else it will return negative error code.
 **/
static int
lpfc_sli4_rb_setup(struct lpfc_hba *phba)
{
	phba->hbq_in_use = 1;
	phba->hbqs[LPFC_ELS_HBQ].entry_count =
		lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count;
	phba->hbq_count = 1;
	lpfc_sli_hbqbuf_init_hbqs(phba, LPFC_ELS_HBQ);
	/* Initially populate or replenish the HBQs */
	return 0;
}

/**
 * lpfc_sli_config_port - Issue config port mailbox command
 * @phba: Pointer to HBA context object.
 * @sli_mode: sli mode - 2/3
 *
 * This function is called by the sli initialization code path
 * to issue config_port mailbox command. This function restarts the
 * HBA firmware and issues a config_port mailbox command to configure
 * the SLI interface in the sli mode specified by sli_mode
 * variable. The caller is not required to hold any locks.
 * The function returns 0 if successful, else returns negative error
 * code.
 **/
int
lpfc_sli_config_port(struct lpfc_hba *phba, int sli_mode)
{
	LPFC_MBOXQ_t *pmb;
	uint32_t resetcount = 0, rc = 0, done = 0;

	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!pmb) {
		phba->link_state = LPFC_HBA_ERROR;
		return -ENOMEM;
	}

	phba->sli_rev = sli_mode;
	while (resetcount < 2 && !done) {
		spin_lock_irq(&phba->hbalock);
		phba->sli.sli_flag |= LPFC_SLI_MBOX_ACTIVE;
		spin_unlock_irq(&phba->hbalock);
		phba->pport->port_state = LPFC_VPORT_UNKNOWN;
		lpfc_sli_brdrestart(phba);
		rc = lpfc_sli_chipset_init(phba);
		if (rc)
			break;

		spin_lock_irq(&phba->hbalock);
		phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
		spin_unlock_irq(&phba->hbalock);
		resetcount++;

		/* Call pre CONFIG_PORT mailbox command initialization.  A
		 * value of 0 means the call was successful.  Any other
		 * nonzero value is a failure, but if ERESTART is returned,
		 * the driver may reset the HBA and try again.
		 */
		rc = lpfc_config_port_prep(phba);
		if (rc == -ERESTART) {
			phba->link_state = LPFC_LINK_UNKNOWN;
			continue;
		} else if (rc)
			break;

		phba->link_state = LPFC_INIT_MBX_CMDS;
		lpfc_config_port(phba, pmb);
		rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
		phba->sli3_options &= ~(LPFC_SLI3_NPIV_ENABLED |
					LPFC_SLI3_HBQ_ENABLED |
					LPFC_SLI3_CRP_ENABLED |
					LPFC_SLI3_DSS_ENABLED);
		if (rc != MBX_SUCCESS) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"0442 Adapter failed to init, mbxCmd x%x "
				"CONFIG_PORT, mbxStatus x%x Data: x%x\n",
				pmb->u.mb.mbxCommand, pmb->u.mb.mbxStatus, 0);
			spin_lock_irq(&phba->hbalock);
			phba->sli.sli_flag &= ~LPFC_SLI_ACTIVE;
			spin_unlock_irq(&phba->hbalock);
			rc = -ENXIO;
		} else {
			/* Allow asynchronous mailbox command to go through */
			spin_lock_irq(&phba->hbalock);
			phba->sli.sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
			spin_unlock_irq(&phba->hbalock);
			done = 1;

			if ((pmb->u.mb.un.varCfgPort.casabt == 1) &&
			    (pmb->u.mb.un.varCfgPort.gasabt == 0))
				lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
					"3110 Port did not grant ASABT\n");
		}
	}
	if (!done) {
		rc = -EINVAL;
		goto do_prep_failed;
	}
	if (pmb->u.mb.un.varCfgPort.sli_mode == 3) {
		if (!pmb->u.mb.un.varCfgPort.cMA) {
			rc = -ENXIO;
			goto do_prep_failed;
		}
		if (phba->max_vpi && pmb->u.mb.un.varCfgPort.gmv) {
			phba->sli3_options |= LPFC_SLI3_NPIV_ENABLED;
			phba->max_vpi = pmb->u.mb.un.varCfgPort.max_vpi;
			phba->max_vports = (phba->max_vpi > phba->max_vports) ?
				phba->max_vpi : phba->max_vports;

		} else
			phba->max_vpi = 0;
		phba->fips_level = 0;
		phba->fips_spec_rev = 0;
		if (pmb->u.mb.un.varCfgPort.gdss) {
			phba->sli3_options |= LPFC_SLI3_DSS_ENABLED;
			phba->fips_level = pmb->u.mb.un.varCfgPort.fips_level;
			phba->fips_spec_rev = pmb->u.mb.un.varCfgPort.fips_rev;
			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
					"2850 Security Crypto Active. FIPS x%d "
					"(Spec Rev: x%d)",
					phba->fips_level, phba->fips_spec_rev);
		}
		if (pmb->u.mb.un.varCfgPort.sec_err) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"2856 Config Port Security Crypto "
					"Error: x%x ",
					pmb->u.mb.un.varCfgPort.sec_err);
		}
		if (pmb->u.mb.un.varCfgPort.gerbm)
			phba->sli3_options |= LPFC_SLI3_HBQ_ENABLED;
		if (pmb->u.mb.un.varCfgPort.gcrp)
			phba->sli3_options |= LPFC_SLI3_CRP_ENABLED;

		phba->hbq_get = phba->mbox->us.s3_pgp.hbq_get;
		phba->port_gp = phba->mbox->us.s3_pgp.port;

		if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
			if (pmb->u.mb.un.varCfgPort.gbg == 0) {
				phba->cfg_enable_bg = 0;
				phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
						"0443 Adapter did not grant "
						"BlockGuard\n");
			}
		}
	} else {
		phba->hbq_get = NULL;
		phba->port_gp = phba->mbox->us.s2.port;
		phba->max_vpi = 0;
	}
do_prep_failed:
	mempool_free(pmb, phba->mbox_mem_pool);
	return rc;
}


/**
 * lpfc_sli_hba_setup - SLI initialization function
 * @phba: Pointer to HBA context object.
 *
 * This function is the main SLI initialization function. This function
 * is called by the HBA initialization code, HBA reset code and HBA
 * error attention handler code. Caller is not required to hold any
 * locks. This function issues config_port mailbox command to configure
 * the SLI, setup iocb rings and HBQ rings. In the end the function
 * calls the config_port_post function to issue init_link mailbox
 * command and to start the discovery. The function will return zero
 * if successful, else it will return negative error code.
 **/
int
lpfc_sli_hba_setup(struct lpfc_hba *phba)
{
	uint32_t rc;
	int  mode = 3, i;
	int longs;

	switch (phba->cfg_sli_mode) {
	case 2:
		if (phba->cfg_enable_npiv) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT,
				"1824 NPIV enabled: Override sli_mode "
				"parameter (%d) to auto (0).\n",
				phba->cfg_sli_mode);
			break;
		}
		mode = 2;
		break;
	case 0:
	case 3:
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT,
				"1819 Unrecognized sli_mode parameter: %d.\n",
				phba->cfg_sli_mode);

		break;
	}
	phba->fcp_embed_io = 0;	/* SLI4 FC support only */

	rc = lpfc_sli_config_port(phba, mode);

	if (rc && phba->cfg_sli_mode == 3)
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT,
				"1820 Unable to select SLI-3.  "
				"Not supported by adapter.\n");
	if (rc && mode != 2)
		rc = lpfc_sli_config_port(phba, 2);
	else if (rc && mode == 2)
		rc = lpfc_sli_config_port(phba, 3);
	if (rc)
		goto lpfc_sli_hba_setup_error;

	/* Enable PCIe device Advanced Error Reporting (AER) if configured */
	if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) {
		rc = pci_enable_pcie_error_reporting(phba->pcidev);
		if (!rc) {
			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
					"2709 This device supports "
					"Advanced Error Reporting (AER)\n");
			spin_lock_irq(&phba->hbalock);
			phba->hba_flag |= HBA_AER_ENABLED;
			spin_unlock_irq(&phba->hbalock);
		} else {
			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
					"2708 This device does not support "
					"Advanced Error Reporting (AER): %d\n",
					rc);
			phba->cfg_aer_support = 0;
		}
	}

	if (phba->sli_rev == 3) {
		phba->iocb_cmd_size = SLI3_IOCB_CMD_SIZE;
		phba->iocb_rsp_size = SLI3_IOCB_RSP_SIZE;
	} else {
		phba->iocb_cmd_size = SLI2_IOCB_CMD_SIZE;
		phba->iocb_rsp_size = SLI2_IOCB_RSP_SIZE;
		phba->sli3_options = 0;
	}

	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
			"0444 Firmware in SLI %x mode. Max_vpi %d\n",
			phba->sli_rev, phba->max_vpi);
	rc = lpfc_sli_ring_map(phba);

	if (rc)
		goto lpfc_sli_hba_setup_error;

	/* Initialize VPIs. */
	if (phba->sli_rev == LPFC_SLI_REV3) {
		/*
		 * The VPI bitmask and physical ID array are allocated
		 * and initialized once only - at driver load.  A port
		 * reset doesn't need to reinitialize this memory.
		 */
		if ((phba->vpi_bmask == NULL) && (phba->vpi_ids == NULL)) {
			longs = (phba->max_vpi + BITS_PER_LONG) / BITS_PER_LONG;
			phba->vpi_bmask = kcalloc(longs,
						  sizeof(unsigned long),
						  GFP_KERNEL);
			if (!phba->vpi_bmask) {
				rc = -ENOMEM;
				goto lpfc_sli_hba_setup_error;
			}

			phba->vpi_ids = kcalloc(phba->max_vpi + 1,
						sizeof(uint16_t),
						GFP_KERNEL);
			if (!phba->vpi_ids) {
				kfree(phba->vpi_bmask);
				rc = -ENOMEM;
				goto lpfc_sli_hba_setup_error;
			}
			for (i = 0; i < phba->max_vpi; i++)
				phba->vpi_ids[i] = i;
		}
	}

	/* Init HBQs */
	if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
		rc = lpfc_sli_hbq_setup(phba);
		if (rc)
			goto lpfc_sli_hba_setup_error;
	}
	spin_lock_irq(&phba->hbalock);
	phba->sli.sli_flag |= LPFC_PROCESS_LA;
	spin_unlock_irq(&phba->hbalock);

	rc = lpfc_config_port_post(phba);
	if (rc)
		goto lpfc_sli_hba_setup_error;

	return rc;

lpfc_sli_hba_setup_error:
	phba->link_state = LPFC_HBA_ERROR;
	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
			"0445 Firmware initialization failed\n");
	return rc;
}

/**
 * lpfc_sli4_read_fcoe_params - Read fcoe params from conf region
 * @phba: Pointer to HBA context object.
 * @mboxq: mailbox pointer.
 * This function issue a dump mailbox command to read config region
 * 23 and parse the records in the region and populate driver
 * data structure.
 **/
static int
lpfc_sli4_read_fcoe_params(struct lpfc_hba *phba)
{
	LPFC_MBOXQ_t *mboxq;
	struct lpfc_dmabuf *mp;
	struct lpfc_mqe *mqe;
	uint32_t data_length;
	int rc;

	/* Program the default value of vlan_id and fc_map */
	phba->valid_vlan = 0;
	phba->fc_map[0] = LPFC_FCOE_FCF_MAP0;
	phba->fc_map[1] = LPFC_FCOE_FCF_MAP1;
	phba->fc_map[2] = LPFC_FCOE_FCF_MAP2;

	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mboxq)
		return -ENOMEM;

	mqe = &mboxq->u.mqe;
	if (lpfc_sli4_dump_cfg_rg23(phba, mboxq)) {
		rc = -ENOMEM;
		goto out_free_mboxq;
	}

	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);

	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
			"(%d):2571 Mailbox cmd x%x Status x%x "
			"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x "
			"x%x x%x x%x x%x x%x x%x x%x x%x x%x "
			"CQ: x%x x%x x%x x%x\n",
			mboxq->vport ? mboxq->vport->vpi : 0,
			bf_get(lpfc_mqe_command, mqe),
			bf_get(lpfc_mqe_status, mqe),
			mqe->un.mb_words[0], mqe->un.mb_words[1],
			mqe->un.mb_words[2], mqe->un.mb_words[3],
			mqe->un.mb_words[4], mqe->un.mb_words[5],
			mqe->un.mb_words[6], mqe->un.mb_words[7],
			mqe->un.mb_words[8], mqe->un.mb_words[9],
			mqe->un.mb_words[10], mqe->un.mb_words[11],
			mqe->un.mb_words[12], mqe->un.mb_words[13],
			mqe->un.mb_words[14], mqe->un.mb_words[15],
			mqe->un.mb_words[16], mqe->un.mb_words[50],
			mboxq->mcqe.word0,
			mboxq->mcqe.mcqe_tag0, 	mboxq->mcqe.mcqe_tag1,
			mboxq->mcqe.trailer);

	if (rc) {
		lpfc_mbuf_free(phba, mp->virt, mp->phys);
		kfree(mp);
		rc = -EIO;
		goto out_free_mboxq;
	}
	data_length = mqe->un.mb_words[5];
	if (data_length > DMP_RGN23_SIZE) {
		lpfc_mbuf_free(phba, mp->virt, mp->phys);
		kfree(mp);
		rc = -EIO;
		goto out_free_mboxq;
	}

	lpfc_parse_fcoe_conf(phba, mp->virt, data_length);
	lpfc_mbuf_free(phba, mp->virt, mp->phys);
	kfree(mp);
	rc = 0;

out_free_mboxq:
	mempool_free(mboxq, phba->mbox_mem_pool);
	return rc;
}

/**
 * lpfc_sli4_read_rev - Issue READ_REV and collect vpd data
 * @phba: pointer to lpfc hba data structure.
 * @mboxq: pointer to the LPFC_MBOXQ_t structure.
 * @vpd: pointer to the memory to hold resulting port vpd data.
 * @vpd_size: On input, the number of bytes allocated to @vpd.
 *	      On output, the number of data bytes in @vpd.
 *
 * This routine executes a READ_REV SLI4 mailbox command.  In
 * addition, this routine gets the port vpd data.
 *
 * Return codes
 * 	0 - successful
 * 	-ENOMEM - could not allocated memory.
 **/
static int
lpfc_sli4_read_rev(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
		    uint8_t *vpd, uint32_t *vpd_size)
{
	int rc = 0;
	uint32_t dma_size;
	struct lpfc_dmabuf *dmabuf;
	struct lpfc_mqe *mqe;

	dmabuf = kzalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
	if (!dmabuf)
		return -ENOMEM;

	/*
	 * Get a DMA buffer for the vpd data resulting from the READ_REV
	 * mailbox command.
	 */
	dma_size = *vpd_size;
	dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev, dma_size,
					  &dmabuf->phys, GFP_KERNEL);
	if (!dmabuf->virt) {
		kfree(dmabuf);
		return -ENOMEM;
	}

	/*
	 * The SLI4 implementation of READ_REV conflicts at word1,
	 * bits 31:16 and SLI4 adds vpd functionality not present
	 * in SLI3.  This code corrects the conflicts.
	 */
	lpfc_read_rev(phba, mboxq);
	mqe = &mboxq->u.mqe;
	mqe->un.read_rev.vpd_paddr_high = putPaddrHigh(dmabuf->phys);
	mqe->un.read_rev.vpd_paddr_low = putPaddrLow(dmabuf->phys);
	mqe->un.read_rev.word1 &= 0x0000FFFF;
	bf_set(lpfc_mbx_rd_rev_vpd, &mqe->un.read_rev, 1);
	bf_set(lpfc_mbx_rd_rev_avail_len, &mqe->un.read_rev, dma_size);

	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
	if (rc) {
		dma_free_coherent(&phba->pcidev->dev, dma_size,
				  dmabuf->virt, dmabuf->phys);
		kfree(dmabuf);
		return -EIO;
	}

	/*
	 * The available vpd length cannot be bigger than the
	 * DMA buffer passed to the port.  Catch the less than
	 * case and update the caller's size.
	 */
	if (mqe->un.read_rev.avail_vpd_len < *vpd_size)
		*vpd_size = mqe->un.read_rev.avail_vpd_len;

	memcpy(vpd, dmabuf->virt, *vpd_size);

	dma_free_coherent(&phba->pcidev->dev, dma_size,
			  dmabuf->virt, dmabuf->phys);
	kfree(dmabuf);
	return 0;
}

/**
 * lpfc_sli4_get_ctl_attr - Retrieve SLI4 device controller attributes
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine retrieves SLI4 device physical port name this PCI function
 * is attached to.
 *
 * Return codes
 *      0 - successful
 *      otherwise - failed to retrieve controller attributes
 **/
static int
lpfc_sli4_get_ctl_attr(struct lpfc_hba *phba)
{
	LPFC_MBOXQ_t *mboxq;
	struct lpfc_mbx_get_cntl_attributes *mbx_cntl_attr;
	struct lpfc_controller_attribute *cntl_attr;
	void *virtaddr = NULL;
	uint32_t alloclen, reqlen;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	int rc;

	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mboxq)
		return -ENOMEM;

	/* Send COMMON_GET_CNTL_ATTRIBUTES mbox cmd */
	reqlen = sizeof(struct lpfc_mbx_get_cntl_attributes);
	alloclen = lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
			LPFC_MBOX_OPCODE_GET_CNTL_ATTRIBUTES, reqlen,
			LPFC_SLI4_MBX_NEMBED);

	if (alloclen < reqlen) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"3084 Allocated DMA memory size (%d) is "
				"less than the requested DMA memory size "
				"(%d)\n", alloclen, reqlen);
		rc = -ENOMEM;
		goto out_free_mboxq;
	}
	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
	virtaddr = mboxq->sge_array->addr[0];
	mbx_cntl_attr = (struct lpfc_mbx_get_cntl_attributes *)virtaddr;
	shdr = &mbx_cntl_attr->cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"3085 Mailbox x%x (x%x/x%x) failed, "
				"rc:x%x, status:x%x, add_status:x%x\n",
				bf_get(lpfc_mqe_command, &mboxq->u.mqe),
				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
				rc, shdr_status, shdr_add_status);
		rc = -ENXIO;
		goto out_free_mboxq;
	}

	cntl_attr = &mbx_cntl_attr->cntl_attr;
	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_VAL;
	phba->sli4_hba.lnk_info.lnk_tp =
		bf_get(lpfc_cntl_attr_lnk_type, cntl_attr);
	phba->sli4_hba.lnk_info.lnk_no =
		bf_get(lpfc_cntl_attr_lnk_numb, cntl_attr);

	memset(phba->BIOSVersion, 0, sizeof(phba->BIOSVersion));
	strlcat(phba->BIOSVersion, (char *)cntl_attr->bios_ver_str,
		sizeof(phba->BIOSVersion));

	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"3086 lnk_type:%d, lnk_numb:%d, bios_ver:%s\n",
			phba->sli4_hba.lnk_info.lnk_tp,
			phba->sli4_hba.lnk_info.lnk_no,
			phba->BIOSVersion);
out_free_mboxq:
	if (rc != MBX_TIMEOUT) {
		if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG)
			lpfc_sli4_mbox_cmd_free(phba, mboxq);
		else
			mempool_free(mboxq, phba->mbox_mem_pool);
	}
	return rc;
}

/**
 * lpfc_sli4_retrieve_pport_name - Retrieve SLI4 device physical port name
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine retrieves SLI4 device physical port name this PCI function
 * is attached to.
 *
 * Return codes
 *      0 - successful
 *      otherwise - failed to retrieve physical port name
 **/
static int
lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba)
{
	LPFC_MBOXQ_t *mboxq;
	struct lpfc_mbx_get_port_name *get_port_name;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	char cport_name = 0;
	int rc;

	/* We assume nothing at this point */
	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_NON;

	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mboxq)
		return -ENOMEM;
	/* obtain link type and link number via READ_CONFIG */
	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
	lpfc_sli4_read_config(phba);
	if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL)
		goto retrieve_ppname;

	/* obtain link type and link number via COMMON_GET_CNTL_ATTRIBUTES */
	rc = lpfc_sli4_get_ctl_attr(phba);
	if (rc)
		goto out_free_mboxq;

retrieve_ppname:
	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
		LPFC_MBOX_OPCODE_GET_PORT_NAME,
		sizeof(struct lpfc_mbx_get_port_name) -
		sizeof(struct lpfc_sli4_cfg_mhdr),
		LPFC_SLI4_MBX_EMBED);
	get_port_name = &mboxq->u.mqe.un.get_port_name;
	shdr = (union lpfc_sli4_cfg_shdr *)&get_port_name->header.cfg_shdr;
	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_OPCODE_VERSION_1);
	bf_set(lpfc_mbx_get_port_name_lnk_type, &get_port_name->u.request,
		phba->sli4_hba.lnk_info.lnk_tp);
	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"3087 Mailbox x%x (x%x/x%x) failed: "
				"rc:x%x, status:x%x, add_status:x%x\n",
				bf_get(lpfc_mqe_command, &mboxq->u.mqe),
				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
				rc, shdr_status, shdr_add_status);
		rc = -ENXIO;
		goto out_free_mboxq;
	}
	switch (phba->sli4_hba.lnk_info.lnk_no) {
	case LPFC_LINK_NUMBER_0:
		cport_name = bf_get(lpfc_mbx_get_port_name_name0,
				&get_port_name->u.response);
		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
		break;
	case LPFC_LINK_NUMBER_1:
		cport_name = bf_get(lpfc_mbx_get_port_name_name1,
				&get_port_name->u.response);
		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
		break;
	case LPFC_LINK_NUMBER_2:
		cport_name = bf_get(lpfc_mbx_get_port_name_name2,
				&get_port_name->u.response);
		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
		break;
	case LPFC_LINK_NUMBER_3:
		cport_name = bf_get(lpfc_mbx_get_port_name_name3,
				&get_port_name->u.response);
		phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_GET;
		break;
	default:
		break;
	}

	if (phba->sli4_hba.pport_name_sta == LPFC_SLI4_PPNAME_GET) {
		phba->Port[0] = cport_name;
		phba->Port[1] = '\0';
		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"3091 SLI get port name: %s\n", phba->Port);
	}

out_free_mboxq:
	if (rc != MBX_TIMEOUT) {
		if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG)
			lpfc_sli4_mbox_cmd_free(phba, mboxq);
		else
			mempool_free(mboxq, phba->mbox_mem_pool);
	}
	return rc;
}

/**
 * lpfc_sli4_arm_cqeq_intr - Arm sli-4 device completion and event queues
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine is called to explicitly arm the SLI4 device's completion and
 * event queues
 **/
static void
lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba)
{
	int qidx;
	struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
	struct lpfc_sli4_hdw_queue *qp;

	sli4_hba->sli4_write_cq_db(phba, sli4_hba->mbx_cq, 0, LPFC_QUEUE_REARM);
	sli4_hba->sli4_write_cq_db(phba, sli4_hba->els_cq, 0, LPFC_QUEUE_REARM);
	if (sli4_hba->nvmels_cq)
		sli4_hba->sli4_write_cq_db(phba, sli4_hba->nvmels_cq, 0,
					   LPFC_QUEUE_REARM);

	qp = sli4_hba->hdwq;
	if (sli4_hba->hdwq) {
		for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) {
			sli4_hba->sli4_write_cq_db(phba, qp[qidx].fcp_cq, 0,
						   LPFC_QUEUE_REARM);
			sli4_hba->sli4_write_cq_db(phba, qp[qidx].nvme_cq, 0,
						   LPFC_QUEUE_REARM);
		}

		for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++)
			sli4_hba->sli4_write_eq_db(phba, qp[qidx].hba_eq,
						0, LPFC_QUEUE_REARM);
	}

	if (phba->nvmet_support) {
		for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) {
			sli4_hba->sli4_write_cq_db(phba,
				sli4_hba->nvmet_cqset[qidx], 0,
				LPFC_QUEUE_REARM);
		}
	}
}

/**
 * lpfc_sli4_get_avail_extnt_rsrc - Get available resource extent count.
 * @phba: Pointer to HBA context object.
 * @type: The resource extent type.
 * @extnt_count: buffer to hold port available extent count.
 * @extnt_size: buffer to hold element count per extent.
 *
 * This function calls the port and retrievs the number of available
 * extents and their size for a particular extent type.
 *
 * Returns: 0 if successful.  Nonzero otherwise.
 **/
int
lpfc_sli4_get_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type,
			       uint16_t *extnt_count, uint16_t *extnt_size)
{
	int rc = 0;
	uint32_t length;
	uint32_t mbox_tmo;
	struct lpfc_mbx_get_rsrc_extent_info *rsrc_info;
	LPFC_MBOXQ_t *mbox;

	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;

	/* Find out how many extents are available for this resource type */
	length = (sizeof(struct lpfc_mbx_get_rsrc_extent_info) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_GET_RSRC_EXTENT_INFO,
			 length, LPFC_SLI4_MBX_EMBED);

	/* Send an extents count of 0 - the GET doesn't use it. */
	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, 0, type,
					LPFC_SLI4_MBX_EMBED);
	if (unlikely(rc)) {
		rc = -EIO;
		goto err_exit;
	}

	if (!phba->sli4_hba.intr_enable)
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	else {
		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
	}
	if (unlikely(rc)) {
		rc = -EIO;
		goto err_exit;
	}

	rsrc_info = &mbox->u.mqe.un.rsrc_extent_info;
	if (bf_get(lpfc_mbox_hdr_status,
		   &rsrc_info->header.cfg_shdr.response)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
				"2930 Failed to get resource extents "
				"Status 0x%x Add'l Status 0x%x\n",
				bf_get(lpfc_mbox_hdr_status,
				       &rsrc_info->header.cfg_shdr.response),
				bf_get(lpfc_mbox_hdr_add_status,
				       &rsrc_info->header.cfg_shdr.response));
		rc = -EIO;
		goto err_exit;
	}

	*extnt_count = bf_get(lpfc_mbx_get_rsrc_extent_info_cnt,
			      &rsrc_info->u.rsp);
	*extnt_size = bf_get(lpfc_mbx_get_rsrc_extent_info_size,
			     &rsrc_info->u.rsp);

	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"3162 Retrieved extents type-%d from port: count:%d, "
			"size:%d\n", type, *extnt_count, *extnt_size);

err_exit:
	mempool_free(mbox, phba->mbox_mem_pool);
	return rc;
}

/**
 * lpfc_sli4_chk_avail_extnt_rsrc - Check for available SLI4 resource extents.
 * @phba: Pointer to HBA context object.
 * @type: The extent type to check.
 *
 * This function reads the current available extents from the port and checks
 * if the extent count or extent size has changed since the last access.
 * Callers use this routine post port reset to understand if there is a
 * extent reprovisioning requirement.
 *
 * Returns:
 *   -Error: error indicates problem.
 *   1: Extent count or size has changed.
 *   0: No changes.
 **/
static int
lpfc_sli4_chk_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type)
{
	uint16_t curr_ext_cnt, rsrc_ext_cnt;
	uint16_t size_diff, rsrc_ext_size;
	int rc = 0;
	struct lpfc_rsrc_blks *rsrc_entry;
	struct list_head *rsrc_blk_list = NULL;

	size_diff = 0;
	curr_ext_cnt = 0;
	rc = lpfc_sli4_get_avail_extnt_rsrc(phba, type,
					    &rsrc_ext_cnt,
					    &rsrc_ext_size);
	if (unlikely(rc))
		return -EIO;

	switch (type) {
	case LPFC_RSC_TYPE_FCOE_RPI:
		rsrc_blk_list = &phba->sli4_hba.lpfc_rpi_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_VPI:
		rsrc_blk_list = &phba->lpfc_vpi_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_XRI:
		rsrc_blk_list = &phba->sli4_hba.lpfc_xri_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_VFI:
		rsrc_blk_list = &phba->sli4_hba.lpfc_vfi_blk_list;
		break;
	default:
		break;
	}

	list_for_each_entry(rsrc_entry, rsrc_blk_list, list) {
		curr_ext_cnt++;
		if (rsrc_entry->rsrc_size != rsrc_ext_size)
			size_diff++;
	}

	if (curr_ext_cnt != rsrc_ext_cnt || size_diff != 0)
		rc = 1;

	return rc;
}

/**
 * lpfc_sli4_cfg_post_extnts -
 * @phba: Pointer to HBA context object.
 * @extnt_cnt - number of available extents.
 * @type - the extent type (rpi, xri, vfi, vpi).
 * @emb - buffer to hold either MBX_EMBED or MBX_NEMBED operation.
 * @mbox - pointer to the caller's allocated mailbox structure.
 *
 * This function executes the extents allocation request.  It also
 * takes care of the amount of memory needed to allocate or get the
 * allocated extents. It is the caller's responsibility to evaluate
 * the response.
 *
 * Returns:
 *   -Error:  Error value describes the condition found.
 *   0: if successful
 **/
static int
lpfc_sli4_cfg_post_extnts(struct lpfc_hba *phba, uint16_t extnt_cnt,
			  uint16_t type, bool *emb, LPFC_MBOXQ_t *mbox)
{
	int rc = 0;
	uint32_t req_len;
	uint32_t emb_len;
	uint32_t alloc_len, mbox_tmo;

	/* Calculate the total requested length of the dma memory */
	req_len = extnt_cnt * sizeof(uint16_t);

	/*
	 * Calculate the size of an embedded mailbox.  The uint32_t
	 * accounts for extents-specific word.
	 */
	emb_len = sizeof(MAILBOX_t) - sizeof(struct mbox_header) -
		sizeof(uint32_t);

	/*
	 * Presume the allocation and response will fit into an embedded
	 * mailbox.  If not true, reconfigure to a non-embedded mailbox.
	 */
	*emb = LPFC_SLI4_MBX_EMBED;
	if (req_len > emb_len) {
		req_len = extnt_cnt * sizeof(uint16_t) +
			sizeof(union lpfc_sli4_cfg_shdr) +
			sizeof(uint32_t);
		*emb = LPFC_SLI4_MBX_NEMBED;
	}

	alloc_len = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
				     LPFC_MBOX_OPCODE_ALLOC_RSRC_EXTENT,
				     req_len, *emb);
	if (alloc_len < req_len) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
			"2982 Allocated DMA memory size (x%x) is "
			"less than the requested DMA memory "
			"size (x%x)\n", alloc_len, req_len);
		return -ENOMEM;
	}
	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, extnt_cnt, type, *emb);
	if (unlikely(rc))
		return -EIO;

	if (!phba->sli4_hba.intr_enable)
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	else {
		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
	}

	if (unlikely(rc))
		rc = -EIO;
	return rc;
}

/**
 * lpfc_sli4_alloc_extent - Allocate an SLI4 resource extent.
 * @phba: Pointer to HBA context object.
 * @type:  The resource extent type to allocate.
 *
 * This function allocates the number of elements for the specified
 * resource type.
 **/
static int
lpfc_sli4_alloc_extent(struct lpfc_hba *phba, uint16_t type)
{
	bool emb = false;
	uint16_t rsrc_id_cnt, rsrc_cnt, rsrc_size;
	uint16_t rsrc_id, rsrc_start, j, k;
	uint16_t *ids;
	int i, rc;
	unsigned long longs;
	unsigned long *bmask;
	struct lpfc_rsrc_blks *rsrc_blks;
	LPFC_MBOXQ_t *mbox;
	uint32_t length;
	struct lpfc_id_range *id_array = NULL;
	void *virtaddr = NULL;
	struct lpfc_mbx_nembed_rsrc_extent *n_rsrc;
	struct lpfc_mbx_alloc_rsrc_extents *rsrc_ext;
	struct list_head *ext_blk_list;

	rc = lpfc_sli4_get_avail_extnt_rsrc(phba, type,
					    &rsrc_cnt,
					    &rsrc_size);
	if (unlikely(rc))
		return -EIO;

	if ((rsrc_cnt == 0) || (rsrc_size == 0)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
			"3009 No available Resource Extents "
			"for resource type 0x%x: Count: 0x%x, "
			"Size 0x%x\n", type, rsrc_cnt,
			rsrc_size);
		return -ENOMEM;
	}

	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_INIT | LOG_SLI,
			"2903 Post resource extents type-0x%x: "
			"count:%d, size %d\n", type, rsrc_cnt, rsrc_size);

	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;

	rc = lpfc_sli4_cfg_post_extnts(phba, rsrc_cnt, type, &emb, mbox);
	if (unlikely(rc)) {
		rc = -EIO;
		goto err_exit;
	}

	/*
	 * Figure out where the response is located.  Then get local pointers
	 * to the response data.  The port does not guarantee to respond to
	 * all extents counts request so update the local variable with the
	 * allocated count from the port.
	 */
	if (emb == LPFC_SLI4_MBX_EMBED) {
		rsrc_ext = &mbox->u.mqe.un.alloc_rsrc_extents;
		id_array = &rsrc_ext->u.rsp.id[0];
		rsrc_cnt = bf_get(lpfc_mbx_rsrc_cnt, &rsrc_ext->u.rsp);
	} else {
		virtaddr = mbox->sge_array->addr[0];
		n_rsrc = (struct lpfc_mbx_nembed_rsrc_extent *) virtaddr;
		rsrc_cnt = bf_get(lpfc_mbx_rsrc_cnt, n_rsrc);
		id_array = &n_rsrc->id;
	}

	longs = ((rsrc_cnt * rsrc_size) + BITS_PER_LONG - 1) / BITS_PER_LONG;
	rsrc_id_cnt = rsrc_cnt * rsrc_size;

	/*
	 * Based on the resource size and count, correct the base and max
	 * resource values.
	 */
	length = sizeof(struct lpfc_rsrc_blks);
	switch (type) {
	case LPFC_RSC_TYPE_FCOE_RPI:
		phba->sli4_hba.rpi_bmask = kcalloc(longs,
						   sizeof(unsigned long),
						   GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.rpi_bmask)) {
			rc = -ENOMEM;
			goto err_exit;
		}
		phba->sli4_hba.rpi_ids = kcalloc(rsrc_id_cnt,
						 sizeof(uint16_t),
						 GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.rpi_ids)) {
			kfree(phba->sli4_hba.rpi_bmask);
			rc = -ENOMEM;
			goto err_exit;
		}

		/*
		 * The next_rpi was initialized with the maximum available
		 * count but the port may allocate a smaller number.  Catch
		 * that case and update the next_rpi.
		 */
		phba->sli4_hba.next_rpi = rsrc_id_cnt;

		/* Initialize local ptrs for common extent processing later. */
		bmask = phba->sli4_hba.rpi_bmask;
		ids = phba->sli4_hba.rpi_ids;
		ext_blk_list = &phba->sli4_hba.lpfc_rpi_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_VPI:
		phba->vpi_bmask = kcalloc(longs, sizeof(unsigned long),
					  GFP_KERNEL);
		if (unlikely(!phba->vpi_bmask)) {
			rc = -ENOMEM;
			goto err_exit;
		}
		phba->vpi_ids = kcalloc(rsrc_id_cnt, sizeof(uint16_t),
					 GFP_KERNEL);
		if (unlikely(!phba->vpi_ids)) {
			kfree(phba->vpi_bmask);
			rc = -ENOMEM;
			goto err_exit;
		}

		/* Initialize local ptrs for common extent processing later. */
		bmask = phba->vpi_bmask;
		ids = phba->vpi_ids;
		ext_blk_list = &phba->lpfc_vpi_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_XRI:
		phba->sli4_hba.xri_bmask = kcalloc(longs,
						   sizeof(unsigned long),
						   GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.xri_bmask)) {
			rc = -ENOMEM;
			goto err_exit;
		}
		phba->sli4_hba.max_cfg_param.xri_used = 0;
		phba->sli4_hba.xri_ids = kcalloc(rsrc_id_cnt,
						 sizeof(uint16_t),
						 GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.xri_ids)) {
			kfree(phba->sli4_hba.xri_bmask);
			rc = -ENOMEM;
			goto err_exit;
		}

		/* Initialize local ptrs for common extent processing later. */
		bmask = phba->sli4_hba.xri_bmask;
		ids = phba->sli4_hba.xri_ids;
		ext_blk_list = &phba->sli4_hba.lpfc_xri_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_VFI:
		phba->sli4_hba.vfi_bmask = kcalloc(longs,
						   sizeof(unsigned long),
						   GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.vfi_bmask)) {
			rc = -ENOMEM;
			goto err_exit;
		}
		phba->sli4_hba.vfi_ids = kcalloc(rsrc_id_cnt,
						 sizeof(uint16_t),
						 GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.vfi_ids)) {
			kfree(phba->sli4_hba.vfi_bmask);
			rc = -ENOMEM;
			goto err_exit;
		}

		/* Initialize local ptrs for common extent processing later. */
		bmask = phba->sli4_hba.vfi_bmask;
		ids = phba->sli4_hba.vfi_ids;
		ext_blk_list = &phba->sli4_hba.lpfc_vfi_blk_list;
		break;
	default:
		/* Unsupported Opcode.  Fail call. */
		id_array = NULL;
		bmask = NULL;
		ids = NULL;
		ext_blk_list = NULL;
		goto err_exit;
	}

	/*
	 * Complete initializing the extent configuration with the
	 * allocated ids assigned to this function.  The bitmask serves
	 * as an index into the array and manages the available ids.  The
	 * array just stores the ids communicated to the port via the wqes.
	 */
	for (i = 0, j = 0, k = 0; i < rsrc_cnt; i++) {
		if ((i % 2) == 0)
			rsrc_id = bf_get(lpfc_mbx_rsrc_id_word4_0,
					 &id_array[k]);
		else
			rsrc_id = bf_get(lpfc_mbx_rsrc_id_word4_1,
					 &id_array[k]);

		rsrc_blks = kzalloc(length, GFP_KERNEL);
		if (unlikely(!rsrc_blks)) {
			rc = -ENOMEM;
			kfree(bmask);
			kfree(ids);
			goto err_exit;
		}
		rsrc_blks->rsrc_start = rsrc_id;
		rsrc_blks->rsrc_size = rsrc_size;
		list_add_tail(&rsrc_blks->list, ext_blk_list);
		rsrc_start = rsrc_id;
		if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0)) {
			phba->sli4_hba.io_xri_start = rsrc_start +
				lpfc_sli4_get_iocb_cnt(phba);
		}

		while (rsrc_id < (rsrc_start + rsrc_size)) {
			ids[j] = rsrc_id;
			rsrc_id++;
			j++;
		}
		/* Entire word processed.  Get next word.*/
		if ((i % 2) == 1)
			k++;
	}
 err_exit:
	lpfc_sli4_mbox_cmd_free(phba, mbox);
	return rc;
}



/**
 * lpfc_sli4_dealloc_extent - Deallocate an SLI4 resource extent.
 * @phba: Pointer to HBA context object.
 * @type: the extent's type.
 *
 * This function deallocates all extents of a particular resource type.
 * SLI4 does not allow for deallocating a particular extent range.  It
 * is the caller's responsibility to release all kernel memory resources.
 **/
static int
lpfc_sli4_dealloc_extent(struct lpfc_hba *phba, uint16_t type)
{
	int rc;
	uint32_t length, mbox_tmo = 0;
	LPFC_MBOXQ_t *mbox;
	struct lpfc_mbx_dealloc_rsrc_extents *dealloc_rsrc;
	struct lpfc_rsrc_blks *rsrc_blk, *rsrc_blk_next;

	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;

	/*
	 * This function sends an embedded mailbox because it only sends the
	 * the resource type.  All extents of this type are released by the
	 * port.
	 */
	length = (sizeof(struct lpfc_mbx_dealloc_rsrc_extents) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_DEALLOC_RSRC_EXTENT,
			 length, LPFC_SLI4_MBX_EMBED);

	/* Send an extents count of 0 - the dealloc doesn't use it. */
	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, 0, type,
					LPFC_SLI4_MBX_EMBED);
	if (unlikely(rc)) {
		rc = -EIO;
		goto out_free_mbox;
	}
	if (!phba->sli4_hba.intr_enable)
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	else {
		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
	}
	if (unlikely(rc)) {
		rc = -EIO;
		goto out_free_mbox;
	}

	dealloc_rsrc = &mbox->u.mqe.un.dealloc_rsrc_extents;
	if (bf_get(lpfc_mbox_hdr_status,
		   &dealloc_rsrc->header.cfg_shdr.response)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
				"2919 Failed to release resource extents "
				"for type %d - Status 0x%x Add'l Status 0x%x. "
				"Resource memory not released.\n",
				type,
				bf_get(lpfc_mbox_hdr_status,
				    &dealloc_rsrc->header.cfg_shdr.response),
				bf_get(lpfc_mbox_hdr_add_status,
				    &dealloc_rsrc->header.cfg_shdr.response));
		rc = -EIO;
		goto out_free_mbox;
	}

	/* Release kernel memory resources for the specific type. */
	switch (type) {
	case LPFC_RSC_TYPE_FCOE_VPI:
		kfree(phba->vpi_bmask);
		kfree(phba->vpi_ids);
		bf_set(lpfc_vpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
				    &phba->lpfc_vpi_blk_list, list) {
			list_del_init(&rsrc_blk->list);
			kfree(rsrc_blk);
		}
		phba->sli4_hba.max_cfg_param.vpi_used = 0;
		break;
	case LPFC_RSC_TYPE_FCOE_XRI:
		kfree(phba->sli4_hba.xri_bmask);
		kfree(phba->sli4_hba.xri_ids);
		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
				    &phba->sli4_hba.lpfc_xri_blk_list, list) {
			list_del_init(&rsrc_blk->list);
			kfree(rsrc_blk);
		}
		break;
	case LPFC_RSC_TYPE_FCOE_VFI:
		kfree(phba->sli4_hba.vfi_bmask);
		kfree(phba->sli4_hba.vfi_ids);
		bf_set(lpfc_vfi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
				    &phba->sli4_hba.lpfc_vfi_blk_list, list) {
			list_del_init(&rsrc_blk->list);
			kfree(rsrc_blk);
		}
		break;
	case LPFC_RSC_TYPE_FCOE_RPI:
		/* RPI bitmask and physical id array are cleaned up earlier. */
		list_for_each_entry_safe(rsrc_blk, rsrc_blk_next,
				    &phba->sli4_hba.lpfc_rpi_blk_list, list) {
			list_del_init(&rsrc_blk->list);
			kfree(rsrc_blk);
		}
		break;
	default:
		break;
	}

	bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);

 out_free_mbox:
	mempool_free(mbox, phba->mbox_mem_pool);
	return rc;
}

static void
lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox,
		  uint32_t feature)
{
	uint32_t len;

	len = sizeof(struct lpfc_mbx_set_feature) -
		sizeof(struct lpfc_sli4_cfg_mhdr);
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_SET_FEATURES, len,
			 LPFC_SLI4_MBX_EMBED);

	switch (feature) {
	case LPFC_SET_UE_RECOVERY:
		bf_set(lpfc_mbx_set_feature_UER,
		       &mbox->u.mqe.un.set_feature, 1);
		mbox->u.mqe.un.set_feature.feature = LPFC_SET_UE_RECOVERY;
		mbox->u.mqe.un.set_feature.param_len = 8;
		break;
	case LPFC_SET_MDS_DIAGS:
		bf_set(lpfc_mbx_set_feature_mds,
		       &mbox->u.mqe.un.set_feature, 1);
		bf_set(lpfc_mbx_set_feature_mds_deep_loopbk,
		       &mbox->u.mqe.un.set_feature, 1);
		mbox->u.mqe.un.set_feature.feature = LPFC_SET_MDS_DIAGS;
		mbox->u.mqe.un.set_feature.param_len = 8;
		break;
	}

	return;
}

/**
 * lpfc_ras_stop_fwlog: Disable FW logging by the adapter
 * @phba: Pointer to HBA context object.
 *
 * Disable FW logging into host memory on the adapter. To
 * be done before reading logs from the host memory.
 **/
void
lpfc_ras_stop_fwlog(struct lpfc_hba *phba)
{
	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;

	ras_fwlog->ras_active = false;

	/* Disable FW logging to host memory */
	writel(LPFC_CTL_PDEV_CTL_DDL_RAS,
	       phba->sli4_hba.conf_regs_memmap_p + LPFC_CTL_PDEV_CTL_OFFSET);
}

/**
 * lpfc_sli4_ras_dma_free - Free memory allocated for FW logging.
 * @phba: Pointer to HBA context object.
 *
 * This function is called to free memory allocated for RAS FW logging
 * support in the driver.
 **/
void
lpfc_sli4_ras_dma_free(struct lpfc_hba *phba)
{
	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
	struct lpfc_dmabuf *dmabuf, *next;

	if (!list_empty(&ras_fwlog->fwlog_buff_list)) {
		list_for_each_entry_safe(dmabuf, next,
				    &ras_fwlog->fwlog_buff_list,
				    list) {
			list_del(&dmabuf->list);
			dma_free_coherent(&phba->pcidev->dev,
					  LPFC_RAS_MAX_ENTRY_SIZE,
					  dmabuf->virt, dmabuf->phys);
			kfree(dmabuf);
		}
	}

	if (ras_fwlog->lwpd.virt) {
		dma_free_coherent(&phba->pcidev->dev,
				  sizeof(uint32_t) * 2,
				  ras_fwlog->lwpd.virt,
				  ras_fwlog->lwpd.phys);
		ras_fwlog->lwpd.virt = NULL;
	}

	ras_fwlog->ras_active = false;
}

/**
 * lpfc_sli4_ras_dma_alloc: Allocate memory for FW support
 * @phba: Pointer to HBA context object.
 * @fwlog_buff_count: Count of buffers to be created.
 *
 * This routine DMA memory for Log Write Position Data[LPWD] and buffer
 * to update FW log is posted to the adapter.
 * Buffer count is calculated based on module param ras_fwlog_buffsize
 * Size of each buffer posted to FW is 64K.
 **/

static int
lpfc_sli4_ras_dma_alloc(struct lpfc_hba *phba,
			uint32_t fwlog_buff_count)
{
	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
	struct lpfc_dmabuf *dmabuf;
	int rc = 0, i = 0;

	/* Initialize List */
	INIT_LIST_HEAD(&ras_fwlog->fwlog_buff_list);

	/* Allocate memory for the LWPD */
	ras_fwlog->lwpd.virt = dma_alloc_coherent(&phba->pcidev->dev,
					    sizeof(uint32_t) * 2,
					    &ras_fwlog->lwpd.phys,
					    GFP_KERNEL);
	if (!ras_fwlog->lwpd.virt) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"6185 LWPD Memory Alloc Failed\n");

		return -ENOMEM;
	}

	ras_fwlog->fw_buffcount = fwlog_buff_count;
	for (i = 0; i < ras_fwlog->fw_buffcount; i++) {
		dmabuf = kzalloc(sizeof(struct lpfc_dmabuf),
				 GFP_KERNEL);
		if (!dmabuf) {
			rc = -ENOMEM;
			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
					"6186 Memory Alloc failed FW logging");
			goto free_mem;
		}

		dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev,
						  LPFC_RAS_MAX_ENTRY_SIZE,
						  &dmabuf->phys, GFP_KERNEL);
		if (!dmabuf->virt) {
			kfree(dmabuf);
			rc = -ENOMEM;
			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
					"6187 DMA Alloc Failed FW logging");
			goto free_mem;
		}
		dmabuf->buffer_tag = i;
		list_add_tail(&dmabuf->list, &ras_fwlog->fwlog_buff_list);
	}

free_mem:
	if (rc)
		lpfc_sli4_ras_dma_free(phba);

	return rc;
}

/**
 * lpfc_sli4_ras_mbox_cmpl: Completion handler for RAS MBX command
 * @phba: pointer to lpfc hba data structure.
 * @pmboxq: pointer to the driver internal queue element for mailbox command.
 *
 * Completion handler for driver's RAS MBX command to the device.
 **/
static void
lpfc_sli4_ras_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
{
	MAILBOX_t *mb;
	union lpfc_sli4_cfg_shdr *shdr;
	uint32_t shdr_status, shdr_add_status;
	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;

	mb = &pmb->u.mb;

	shdr = (union lpfc_sli4_cfg_shdr *)
		&pmb->u.mqe.un.ras_fwlog.header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);

	if (mb->mbxStatus != MBX_SUCCESS || shdr_status) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
				"6188 FW LOG mailbox "
				"completed with status x%x add_status x%x,"
				" mbx status x%x\n",
				shdr_status, shdr_add_status, mb->mbxStatus);

		ras_fwlog->ras_hwsupport = false;
		goto disable_ras;
	}

	ras_fwlog->ras_active = true;
	mempool_free(pmb, phba->mbox_mem_pool);

	return;

disable_ras:
	/* Free RAS DMA memory */
	lpfc_sli4_ras_dma_free(phba);
	mempool_free(pmb, phba->mbox_mem_pool);
}

/**
 * lpfc_sli4_ras_fwlog_init: Initialize memory and post RAS MBX command
 * @phba: pointer to lpfc hba data structure.
 * @fwlog_level: Logging verbosity level.
 * @fwlog_enable: Enable/Disable logging.
 *
 * Initialize memory and post mailbox command to enable FW logging in host
 * memory.
 **/
int
lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba,
			 uint32_t fwlog_level,
			 uint32_t fwlog_enable)
{
	struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog;
	struct lpfc_mbx_set_ras_fwlog *mbx_fwlog = NULL;
	struct lpfc_dmabuf *dmabuf;
	LPFC_MBOXQ_t *mbox;
	uint32_t len = 0, fwlog_buffsize, fwlog_entry_count;
	int rc = 0;

	fwlog_buffsize = (LPFC_RAS_MIN_BUFF_POST_SIZE *
			  phba->cfg_ras_fwlog_buffsize);
	fwlog_entry_count = (fwlog_buffsize/LPFC_RAS_MAX_ENTRY_SIZE);

	/*
	 * If re-enabling FW logging support use earlier allocated
	 * DMA buffers while posting MBX command.
	 **/
	if (!ras_fwlog->lwpd.virt) {
		rc = lpfc_sli4_ras_dma_alloc(phba, fwlog_entry_count);
		if (rc) {
			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
					"6189 FW Log Memory Allocation Failed");
			return rc;
		}
	}

	/* Setup Mailbox command */
	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"6190 RAS MBX Alloc Failed");
		rc = -ENOMEM;
		goto mem_free;
	}

	ras_fwlog->fw_loglevel = fwlog_level;
	len = (sizeof(struct lpfc_mbx_set_ras_fwlog) -
		sizeof(struct lpfc_sli4_cfg_mhdr));

	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_LOWLEVEL,
			 LPFC_MBOX_OPCODE_SET_DIAG_LOG_OPTION,
			 len, LPFC_SLI4_MBX_EMBED);

	mbx_fwlog = (struct lpfc_mbx_set_ras_fwlog *)&mbox->u.mqe.un.ras_fwlog;
	bf_set(lpfc_fwlog_enable, &mbx_fwlog->u.request,
	       fwlog_enable);
	bf_set(lpfc_fwlog_loglvl, &mbx_fwlog->u.request,
	       ras_fwlog->fw_loglevel);
	bf_set(lpfc_fwlog_buffcnt, &mbx_fwlog->u.request,
	       ras_fwlog->fw_buffcount);
	bf_set(lpfc_fwlog_buffsz, &mbx_fwlog->u.request,
	       LPFC_RAS_MAX_ENTRY_SIZE/SLI4_PAGE_SIZE);

	/* Update DMA buffer address */
	list_for_each_entry(dmabuf, &ras_fwlog->fwlog_buff_list, list) {
		memset(dmabuf->virt, 0, LPFC_RAS_MAX_ENTRY_SIZE);

		mbx_fwlog->u.request.buff_fwlog[dmabuf->buffer_tag].addr_lo =
			putPaddrLow(dmabuf->phys);

		mbx_fwlog->u.request.buff_fwlog[dmabuf->buffer_tag].addr_hi =
			putPaddrHigh(dmabuf->phys);
	}

	/* Update LPWD address */
	mbx_fwlog->u.request.lwpd.addr_lo = putPaddrLow(ras_fwlog->lwpd.phys);
	mbx_fwlog->u.request.lwpd.addr_hi = putPaddrHigh(ras_fwlog->lwpd.phys);

	mbox->vport = phba->pport;
	mbox->mbox_cmpl = lpfc_sli4_ras_mbox_cmpl;

	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);

	if (rc == MBX_NOT_FINISHED) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"6191 FW-Log Mailbox failed. "
				"status %d mbxStatus : x%x", rc,
				bf_get(lpfc_mqe_status, &mbox->u.mqe));
		mempool_free(mbox, phba->mbox_mem_pool);
		rc = -EIO;
		goto mem_free;
	} else
		rc = 0;
mem_free:
	if (rc)
		lpfc_sli4_ras_dma_free(phba);

	return rc;
}

/**
 * lpfc_sli4_ras_setup - Check if RAS supported on the adapter
 * @phba: Pointer to HBA context object.
 *
 * Check if RAS is supported on the adapter and initialize it.
 **/
void
lpfc_sli4_ras_setup(struct lpfc_hba *phba)
{
	/* Check RAS FW Log needs to be enabled or not */
	if (lpfc_check_fwlog_support(phba))
		return;

	lpfc_sli4_ras_fwlog_init(phba, phba->cfg_ras_fwlog_level,
				 LPFC_RAS_ENABLE_LOGGING);
}

/**
 * lpfc_sli4_alloc_resource_identifiers - Allocate all SLI4 resource extents.
 * @phba: Pointer to HBA context object.
 *
 * This function allocates all SLI4 resource identifiers.
 **/
int
lpfc_sli4_alloc_resource_identifiers(struct lpfc_hba *phba)
{
	int i, rc, error = 0;
	uint16_t count, base;
	unsigned long longs;

	if (!phba->sli4_hba.rpi_hdrs_in_use)
		phba->sli4_hba.next_rpi = phba->sli4_hba.max_cfg_param.max_rpi;
	if (phba->sli4_hba.extents_in_use) {
		/*
		 * The port supports resource extents. The XRI, VPI, VFI, RPI
		 * resource extent count must be read and allocated before
		 * provisioning the resource id arrays.
		 */
		if (bf_get(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags) ==
		    LPFC_IDX_RSRC_RDY) {
			/*
			 * Extent-based resources are set - the driver could
			 * be in a port reset. Figure out if any corrective
			 * actions need to be taken.
			 */
			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
						 LPFC_RSC_TYPE_FCOE_VFI);
			if (rc != 0)
				error++;
			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
						 LPFC_RSC_TYPE_FCOE_VPI);
			if (rc != 0)
				error++;
			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
						 LPFC_RSC_TYPE_FCOE_XRI);
			if (rc != 0)
				error++;
			rc = lpfc_sli4_chk_avail_extnt_rsrc(phba,
						 LPFC_RSC_TYPE_FCOE_RPI);
			if (rc != 0)
				error++;

			/*
			 * It's possible that the number of resources
			 * provided to this port instance changed between
			 * resets.  Detect this condition and reallocate
			 * resources.  Otherwise, there is no action.
			 */
			if (error) {
				lpfc_printf_log(phba, KERN_INFO,
						LOG_MBOX | LOG_INIT,
						"2931 Detected extent resource "
						"change.  Reallocating all "
						"extents.\n");
				rc = lpfc_sli4_dealloc_extent(phba,
						 LPFC_RSC_TYPE_FCOE_VFI);
				rc = lpfc_sli4_dealloc_extent(phba,
						 LPFC_RSC_TYPE_FCOE_VPI);
				rc = lpfc_sli4_dealloc_extent(phba,
						 LPFC_RSC_TYPE_FCOE_XRI);
				rc = lpfc_sli4_dealloc_extent(phba,
						 LPFC_RSC_TYPE_FCOE_RPI);
			} else
				return 0;
		}

		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_VFI);
		if (unlikely(rc))
			goto err_exit;

		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_VPI);
		if (unlikely(rc))
			goto err_exit;

		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_RPI);
		if (unlikely(rc))
			goto err_exit;

		rc = lpfc_sli4_alloc_extent(phba, LPFC_RSC_TYPE_FCOE_XRI);
		if (unlikely(rc))
			goto err_exit;
		bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags,
		       LPFC_IDX_RSRC_RDY);
		return rc;
	} else {
		/*
		 * The port does not support resource extents.  The XRI, VPI,
		 * VFI, RPI resource ids were determined from READ_CONFIG.
		 * Just allocate the bitmasks and provision the resource id
		 * arrays.  If a port reset is active, the resources don't
		 * need any action - just exit.
		 */
		if (bf_get(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags) ==
		    LPFC_IDX_RSRC_RDY) {
			lpfc_sli4_dealloc_resource_identifiers(phba);
			lpfc_sli4_remove_rpis(phba);
		}
		/* RPIs. */
		count = phba->sli4_hba.max_cfg_param.max_rpi;
		if (count <= 0) {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"3279 Invalid provisioning of "
					"rpi:%d\n", count);
			rc = -EINVAL;
			goto err_exit;
		}
		base = phba->sli4_hba.max_cfg_param.rpi_base;
		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
		phba->sli4_hba.rpi_bmask = kcalloc(longs,
						   sizeof(unsigned long),
						   GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.rpi_bmask)) {
			rc = -ENOMEM;
			goto err_exit;
		}
		phba->sli4_hba.rpi_ids = kcalloc(count, sizeof(uint16_t),
						 GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.rpi_ids)) {
			rc = -ENOMEM;
			goto free_rpi_bmask;
		}

		for (i = 0; i < count; i++)
			phba->sli4_hba.rpi_ids[i] = base + i;

		/* VPIs. */
		count = phba->sli4_hba.max_cfg_param.max_vpi;
		if (count <= 0) {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"3280 Invalid provisioning of "
					"vpi:%d\n", count);
			rc = -EINVAL;
			goto free_rpi_ids;
		}
		base = phba->sli4_hba.max_cfg_param.vpi_base;
		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
		phba->vpi_bmask = kcalloc(longs, sizeof(unsigned long),
					  GFP_KERNEL);
		if (unlikely(!phba->vpi_bmask)) {
			rc = -ENOMEM;
			goto free_rpi_ids;
		}
		phba->vpi_ids = kcalloc(count, sizeof(uint16_t),
					GFP_KERNEL);
		if (unlikely(!phba->vpi_ids)) {
			rc = -ENOMEM;
			goto free_vpi_bmask;
		}

		for (i = 0; i < count; i++)
			phba->vpi_ids[i] = base + i;

		/* XRIs. */
		count = phba->sli4_hba.max_cfg_param.max_xri;
		if (count <= 0) {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"3281 Invalid provisioning of "
					"xri:%d\n", count);
			rc = -EINVAL;
			goto free_vpi_ids;
		}
		base = phba->sli4_hba.max_cfg_param.xri_base;
		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
		phba->sli4_hba.xri_bmask = kcalloc(longs,
						   sizeof(unsigned long),
						   GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.xri_bmask)) {
			rc = -ENOMEM;
			goto free_vpi_ids;
		}
		phba->sli4_hba.max_cfg_param.xri_used = 0;
		phba->sli4_hba.xri_ids = kcalloc(count, sizeof(uint16_t),
						 GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.xri_ids)) {
			rc = -ENOMEM;
			goto free_xri_bmask;
		}

		for (i = 0; i < count; i++)
			phba->sli4_hba.xri_ids[i] = base + i;

		/* VFIs. */
		count = phba->sli4_hba.max_cfg_param.max_vfi;
		if (count <= 0) {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"3282 Invalid provisioning of "
					"vfi:%d\n", count);
			rc = -EINVAL;
			goto free_xri_ids;
		}
		base = phba->sli4_hba.max_cfg_param.vfi_base;
		longs = (count + BITS_PER_LONG - 1) / BITS_PER_LONG;
		phba->sli4_hba.vfi_bmask = kcalloc(longs,
						   sizeof(unsigned long),
						   GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.vfi_bmask)) {
			rc = -ENOMEM;
			goto free_xri_ids;
		}
		phba->sli4_hba.vfi_ids = kcalloc(count, sizeof(uint16_t),
						 GFP_KERNEL);
		if (unlikely(!phba->sli4_hba.vfi_ids)) {
			rc = -ENOMEM;
			goto free_vfi_bmask;
		}

		for (i = 0; i < count; i++)
			phba->sli4_hba.vfi_ids[i] = base + i;

		/*
		 * Mark all resources ready.  An HBA reset doesn't need
		 * to reset the initialization.
		 */
		bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags,
		       LPFC_IDX_RSRC_RDY);
		return 0;
	}

 free_vfi_bmask:
	kfree(phba->sli4_hba.vfi_bmask);
	phba->sli4_hba.vfi_bmask = NULL;
 free_xri_ids:
	kfree(phba->sli4_hba.xri_ids);
	phba->sli4_hba.xri_ids = NULL;
 free_xri_bmask:
	kfree(phba->sli4_hba.xri_bmask);
	phba->sli4_hba.xri_bmask = NULL;
 free_vpi_ids:
	kfree(phba->vpi_ids);
	phba->vpi_ids = NULL;
 free_vpi_bmask:
	kfree(phba->vpi_bmask);
	phba->vpi_bmask = NULL;
 free_rpi_ids:
	kfree(phba->sli4_hba.rpi_ids);
	phba->sli4_hba.rpi_ids = NULL;
 free_rpi_bmask:
	kfree(phba->sli4_hba.rpi_bmask);
	phba->sli4_hba.rpi_bmask = NULL;
 err_exit:
	return rc;
}

/**
 * lpfc_sli4_dealloc_resource_identifiers - Deallocate all SLI4 resource extents.
 * @phba: Pointer to HBA context object.
 *
 * This function allocates the number of elements for the specified
 * resource type.
 **/
int
lpfc_sli4_dealloc_resource_identifiers(struct lpfc_hba *phba)
{
	if (phba->sli4_hba.extents_in_use) {
		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_VPI);
		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_RPI);
		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_XRI);
		lpfc_sli4_dealloc_extent(phba, LPFC_RSC_TYPE_FCOE_VFI);
	} else {
		kfree(phba->vpi_bmask);
		phba->sli4_hba.max_cfg_param.vpi_used = 0;
		kfree(phba->vpi_ids);
		bf_set(lpfc_vpi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
		kfree(phba->sli4_hba.xri_bmask);
		kfree(phba->sli4_hba.xri_ids);
		kfree(phba->sli4_hba.vfi_bmask);
		kfree(phba->sli4_hba.vfi_ids);
		bf_set(lpfc_vfi_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
		bf_set(lpfc_idx_rsrc_rdy, &phba->sli4_hba.sli4_flags, 0);
	}

	return 0;
}

/**
 * lpfc_sli4_get_allocated_extnts - Get the port's allocated extents.
 * @phba: Pointer to HBA context object.
 * @type: The resource extent type.
 * @extnt_count: buffer to hold port extent count response
 * @extnt_size: buffer to hold port extent size response.
 *
 * This function calls the port to read the host allocated extents
 * for a particular type.
 **/
int
lpfc_sli4_get_allocated_extnts(struct lpfc_hba *phba, uint16_t type,
			       uint16_t *extnt_cnt, uint16_t *extnt_size)
{
	bool emb;
	int rc = 0;
	uint16_t curr_blks = 0;
	uint32_t req_len, emb_len;
	uint32_t alloc_len, mbox_tmo;
	struct list_head *blk_list_head;
	struct lpfc_rsrc_blks *rsrc_blk;
	LPFC_MBOXQ_t *mbox;
	void *virtaddr = NULL;
	struct lpfc_mbx_nembed_rsrc_extent *n_rsrc;
	struct lpfc_mbx_alloc_rsrc_extents *rsrc_ext;
	union  lpfc_sli4_cfg_shdr *shdr;

	switch (type) {
	case LPFC_RSC_TYPE_FCOE_VPI:
		blk_list_head = &phba->lpfc_vpi_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_XRI:
		blk_list_head = &phba->sli4_hba.lpfc_xri_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_VFI:
		blk_list_head = &phba->sli4_hba.lpfc_vfi_blk_list;
		break;
	case LPFC_RSC_TYPE_FCOE_RPI:
		blk_list_head = &phba->sli4_hba.lpfc_rpi_blk_list;
		break;
	default:
		return -EIO;
	}

	/* Count the number of extents currently allocatd for this type. */
	list_for_each_entry(rsrc_blk, blk_list_head, list) {
		if (curr_blks == 0) {
			/*
			 * The GET_ALLOCATED mailbox does not return the size,
			 * just the count.  The size should be just the size
			 * stored in the current allocated block and all sizes
			 * for an extent type are the same so set the return
			 * value now.
			 */
			*extnt_size = rsrc_blk->rsrc_size;
		}
		curr_blks++;
	}

	/*
	 * Calculate the size of an embedded mailbox.  The uint32_t
	 * accounts for extents-specific word.
	 */
	emb_len = sizeof(MAILBOX_t) - sizeof(struct mbox_header) -
		sizeof(uint32_t);

	/*
	 * Presume the allocation and response will fit into an embedded
	 * mailbox.  If not true, reconfigure to a non-embedded mailbox.
	 */
	emb = LPFC_SLI4_MBX_EMBED;
	req_len = emb_len;
	if (req_len > emb_len) {
		req_len = curr_blks * sizeof(uint16_t) +
			sizeof(union lpfc_sli4_cfg_shdr) +
			sizeof(uint32_t);
		emb = LPFC_SLI4_MBX_NEMBED;
	}

	mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	memset(mbox, 0, sizeof(LPFC_MBOXQ_t));

	alloc_len = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
				     LPFC_MBOX_OPCODE_GET_ALLOC_RSRC_EXTENT,
				     req_len, emb);
	if (alloc_len < req_len) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
			"2983 Allocated DMA memory size (x%x) is "
			"less than the requested DMA memory "
			"size (x%x)\n", alloc_len, req_len);
		rc = -ENOMEM;
		goto err_exit;
	}
	rc = lpfc_sli4_mbox_rsrc_extent(phba, mbox, curr_blks, type, emb);
	if (unlikely(rc)) {
		rc = -EIO;
		goto err_exit;
	}

	if (!phba->sli4_hba.intr_enable)
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	else {
		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
	}

	if (unlikely(rc)) {
		rc = -EIO;
		goto err_exit;
	}

	/*
	 * Figure out where the response is located.  Then get local pointers
	 * to the response data.  The port does not guarantee to respond to
	 * all extents counts request so update the local variable with the
	 * allocated count from the port.
	 */
	if (emb == LPFC_SLI4_MBX_EMBED) {
		rsrc_ext = &mbox->u.mqe.un.alloc_rsrc_extents;
		shdr = &rsrc_ext->header.cfg_shdr;
		*extnt_cnt = bf_get(lpfc_mbx_rsrc_cnt, &rsrc_ext->u.rsp);
	} else {
		virtaddr = mbox->sge_array->addr[0];
		n_rsrc = (struct lpfc_mbx_nembed_rsrc_extent *) virtaddr;
		shdr = &n_rsrc->cfg_shdr;
		*extnt_cnt = bf_get(lpfc_mbx_rsrc_cnt, n_rsrc);
	}

	if (bf_get(lpfc_mbox_hdr_status, &shdr->response)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT,
			"2984 Failed to read allocated resources "
			"for type %d - Status 0x%x Add'l Status 0x%x.\n",
			type,
			bf_get(lpfc_mbox_hdr_status, &shdr->response),
			bf_get(lpfc_mbox_hdr_add_status, &shdr->response));
		rc = -EIO;
		goto err_exit;
	}
 err_exit:
	lpfc_sli4_mbox_cmd_free(phba, mbox);
	return rc;
}

/**
 * lpfc_sli4_repost_sgl_list - Repost the buffers sgl pages as block
 * @phba: pointer to lpfc hba data structure.
 * @pring: Pointer to driver SLI ring object.
 * @sgl_list: linked link of sgl buffers to post
 * @cnt: number of linked list buffers
 *
 * This routine walks the list of buffers that have been allocated and
 * repost them to the port by using SGL block post. This is needed after a
 * pci_function_reset/warm_start or start. It attempts to construct blocks
 * of buffer sgls which contains contiguous xris and uses the non-embedded
 * SGL block post mailbox commands to post them to the port. For single
 * buffer sgl with non-contiguous xri, if any, it shall use embedded SGL post
 * mailbox command for posting.
 *
 * Returns: 0 = success, non-zero failure.
 **/
static int
lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba,
			  struct list_head *sgl_list, int cnt)
{
	struct lpfc_sglq *sglq_entry = NULL;
	struct lpfc_sglq *sglq_entry_next = NULL;
	struct lpfc_sglq *sglq_entry_first = NULL;
	int status, total_cnt;
	int post_cnt = 0, num_posted = 0, block_cnt = 0;
	int last_xritag = NO_XRI;
	LIST_HEAD(prep_sgl_list);
	LIST_HEAD(blck_sgl_list);
	LIST_HEAD(allc_sgl_list);
	LIST_HEAD(post_sgl_list);
	LIST_HEAD(free_sgl_list);

	spin_lock_irq(&phba->hbalock);
	spin_lock(&phba->sli4_hba.sgl_list_lock);
	list_splice_init(sgl_list, &allc_sgl_list);
	spin_unlock(&phba->sli4_hba.sgl_list_lock);
	spin_unlock_irq(&phba->hbalock);

	total_cnt = cnt;
	list_for_each_entry_safe(sglq_entry, sglq_entry_next,
				 &allc_sgl_list, list) {
		list_del_init(&sglq_entry->list);
		block_cnt++;
		if ((last_xritag != NO_XRI) &&
		    (sglq_entry->sli4_xritag != last_xritag + 1)) {
			/* a hole in xri block, form a sgl posting block */
			list_splice_init(&prep_sgl_list, &blck_sgl_list);
			post_cnt = block_cnt - 1;
			/* prepare list for next posting block */
			list_add_tail(&sglq_entry->list, &prep_sgl_list);
			block_cnt = 1;
		} else {
			/* prepare list for next posting block */
			list_add_tail(&sglq_entry->list, &prep_sgl_list);
			/* enough sgls for non-embed sgl mbox command */
			if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) {
				list_splice_init(&prep_sgl_list,
						 &blck_sgl_list);
				post_cnt = block_cnt;
				block_cnt = 0;
			}
		}
		num_posted++;

		/* keep track of last sgl's xritag */
		last_xritag = sglq_entry->sli4_xritag;

		/* end of repost sgl list condition for buffers */
		if (num_posted == total_cnt) {
			if (post_cnt == 0) {
				list_splice_init(&prep_sgl_list,
						 &blck_sgl_list);
				post_cnt = block_cnt;
			} else if (block_cnt == 1) {
				status = lpfc_sli4_post_sgl(phba,
						sglq_entry->phys, 0,
						sglq_entry->sli4_xritag);
				if (!status) {
					/* successful, put sgl to posted list */
					list_add_tail(&sglq_entry->list,
						      &post_sgl_list);
				} else {
					/* Failure, put sgl to free list */
					lpfc_printf_log(phba, KERN_WARNING,
						LOG_SLI,
						"3159 Failed to post "
						"sgl, xritag:x%x\n",
						sglq_entry->sli4_xritag);
					list_add_tail(&sglq_entry->list,
						      &free_sgl_list);
					total_cnt--;
				}
			}
		}

		/* continue until a nembed page worth of sgls */
		if (post_cnt == 0)
			continue;

		/* post the buffer list sgls as a block */
		status = lpfc_sli4_post_sgl_list(phba, &blck_sgl_list,
						 post_cnt);

		if (!status) {
			/* success, put sgl list to posted sgl list */
			list_splice_init(&blck_sgl_list, &post_sgl_list);
		} else {
			/* Failure, put sgl list to free sgl list */
			sglq_entry_first = list_first_entry(&blck_sgl_list,
							    struct lpfc_sglq,
							    list);
			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
					"3160 Failed to post sgl-list, "
					"xritag:x%x-x%x\n",
					sglq_entry_first->sli4_xritag,
					(sglq_entry_first->sli4_xritag +
					 post_cnt - 1));
			list_splice_init(&blck_sgl_list, &free_sgl_list);
			total_cnt -= post_cnt;
		}

		/* don't reset xirtag due to hole in xri block */
		if (block_cnt == 0)
			last_xritag = NO_XRI;

		/* reset sgl post count for next round of posting */
		post_cnt = 0;
	}

	/* free the sgls failed to post */
	lpfc_free_sgl_list(phba, &free_sgl_list);

	/* push sgls posted to the available list */
	if (!list_empty(&post_sgl_list)) {
		spin_lock_irq(&phba->hbalock);
		spin_lock(&phba->sli4_hba.sgl_list_lock);
		list_splice_init(&post_sgl_list, sgl_list);
		spin_unlock(&phba->sli4_hba.sgl_list_lock);
		spin_unlock_irq(&phba->hbalock);
	} else {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"3161 Failure to post sgl to port.\n");
		return -EIO;
	}

	/* return the number of XRIs actually posted */
	return total_cnt;
}

/**
 * lpfc_sli4_repost_io_sgl_list - Repost all the allocated nvme buffer sgls
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine walks the list of nvme buffers that have been allocated and
 * repost them to the port by using SGL block post. This is needed after a
 * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
 * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
 * to the lpfc_io_buf_list. If the repost fails, reject all nvme buffers.
 *
 * Returns: 0 = success, non-zero failure.
 **/
static int
lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba)
{
	LIST_HEAD(post_nblist);
	int num_posted, rc = 0;

	/* get all NVME buffers need to repost to a local list */
	lpfc_io_buf_flush(phba, &post_nblist);

	/* post the list of nvme buffer sgls to port if available */
	if (!list_empty(&post_nblist)) {
		num_posted = lpfc_sli4_post_io_sgl_list(
			phba, &post_nblist, phba->sli4_hba.io_xri_cnt);
		/* failed to post any nvme buffer, return error */
		if (num_posted == 0)
			rc = -EIO;
	}
	return rc;
}

static void
lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
{
	uint32_t len;

	len = sizeof(struct lpfc_mbx_set_host_data) -
		sizeof(struct lpfc_sli4_cfg_mhdr);
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_SET_HOST_DATA, len,
			 LPFC_SLI4_MBX_EMBED);

	mbox->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_OS_DRIVER_VERSION;
	mbox->u.mqe.un.set_host_data.param_len =
					LPFC_HOST_OS_DRIVER_VERSION_SIZE;
	snprintf(mbox->u.mqe.un.set_host_data.data,
		 LPFC_HOST_OS_DRIVER_VERSION_SIZE,
		 "Linux %s v"LPFC_DRIVER_VERSION,
		 (phba->hba_flag & HBA_FCOE_MODE) ? "FCoE" : "FC");
}

int
lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,
		    struct lpfc_queue *drq, int count, int idx)
{
	int rc, i;
	struct lpfc_rqe hrqe;
	struct lpfc_rqe drqe;
	struct lpfc_rqb *rqbp;
	unsigned long flags;
	struct rqb_dmabuf *rqb_buffer;
	LIST_HEAD(rqb_buf_list);

	spin_lock_irqsave(&phba->hbalock, flags);
	rqbp = hrq->rqbp;
	for (i = 0; i < count; i++) {
		/* IF RQ is already full, don't bother */
		if (rqbp->buffer_count + i >= rqbp->entry_count - 1)
			break;
		rqb_buffer = rqbp->rqb_alloc_buffer(phba);
		if (!rqb_buffer)
			break;
		rqb_buffer->hrq = hrq;
		rqb_buffer->drq = drq;
		rqb_buffer->idx = idx;
		list_add_tail(&rqb_buffer->hbuf.list, &rqb_buf_list);
	}
	while (!list_empty(&rqb_buf_list)) {
		list_remove_head(&rqb_buf_list, rqb_buffer, struct rqb_dmabuf,
				 hbuf.list);

		hrqe.address_lo = putPaddrLow(rqb_buffer->hbuf.phys);
		hrqe.address_hi = putPaddrHigh(rqb_buffer->hbuf.phys);
		drqe.address_lo = putPaddrLow(rqb_buffer->dbuf.phys);
		drqe.address_hi = putPaddrHigh(rqb_buffer->dbuf.phys);
		rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
		if (rc < 0) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"6421 Cannot post to HRQ %d: %x %x %x "
					"DRQ %x %x\n",
					hrq->queue_id,
					hrq->host_index,
					hrq->hba_index,
					hrq->entry_count,
					drq->host_index,
					drq->hba_index);
			rqbp->rqb_free_buffer(phba, rqb_buffer);
		} else {
			list_add_tail(&rqb_buffer->hbuf.list,
				      &rqbp->rqb_buffer_list);
			rqbp->buffer_count++;
		}
	}
	spin_unlock_irqrestore(&phba->hbalock, flags);
	return 1;
}

/**
 * lpfc_sli4_hba_setup - SLI4 device initialization PCI function
 * @phba: Pointer to HBA context object.
 *
 * This function is the main SLI4 device initialization PCI function. This
 * function is called by the HBA initialization code, HBA reset code and
 * HBA error attention handler code. Caller is not required to hold any
 * locks.
 **/
int
lpfc_sli4_hba_setup(struct lpfc_hba *phba)
{
	int rc, i, cnt, len;
	LPFC_MBOXQ_t *mboxq;
	struct lpfc_mqe *mqe;
	uint8_t *vpd;
	uint32_t vpd_size;
	uint32_t ftr_rsp = 0;
	struct Scsi_Host *shost = lpfc_shost_from_vport(phba->pport);
	struct lpfc_vport *vport = phba->pport;
	struct lpfc_dmabuf *mp;
	struct lpfc_rqb *rqbp;

	/* Perform a PCI function reset to start from clean */
	rc = lpfc_pci_function_reset(phba);
	if (unlikely(rc))
		return -ENODEV;

	/* Check the HBA Host Status Register for readyness */
	rc = lpfc_sli4_post_status_check(phba);
	if (unlikely(rc))
		return -ENODEV;
	else {
		spin_lock_irq(&phba->hbalock);
		phba->sli.sli_flag |= LPFC_SLI_ACTIVE;
		spin_unlock_irq(&phba->hbalock);
	}

	/*
	 * Allocate a single mailbox container for initializing the
	 * port.
	 */
	mboxq = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mboxq)
		return -ENOMEM;

	/* Issue READ_REV to collect vpd and FW information. */
	vpd_size = SLI4_PAGE_SIZE;
	vpd = kzalloc(vpd_size, GFP_KERNEL);
	if (!vpd) {
		rc = -ENOMEM;
		goto out_free_mbox;
	}

	rc = lpfc_sli4_read_rev(phba, mboxq, vpd, &vpd_size);
	if (unlikely(rc)) {
		kfree(vpd);
		goto out_free_mbox;
	}

	mqe = &mboxq->u.mqe;
	phba->sli_rev = bf_get(lpfc_mbx_rd_rev_sli_lvl, &mqe->un.read_rev);
	if (bf_get(lpfc_mbx_rd_rev_fcoe, &mqe->un.read_rev)) {
		phba->hba_flag |= HBA_FCOE_MODE;
		phba->fcp_embed_io = 0;	/* SLI4 FC support only */
	} else {
		phba->hba_flag &= ~HBA_FCOE_MODE;
	}

	if (bf_get(lpfc_mbx_rd_rev_cee_ver, &mqe->un.read_rev) ==
		LPFC_DCBX_CEE_MODE)
		phba->hba_flag |= HBA_FIP_SUPPORT;
	else
		phba->hba_flag &= ~HBA_FIP_SUPPORT;

	phba->hba_flag &= ~HBA_FCP_IOQ_FLUSH;

	if (phba->sli_rev != LPFC_SLI_REV4) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
			"0376 READ_REV Error. SLI Level %d "
			"FCoE enabled %d\n",
			phba->sli_rev, phba->hba_flag & HBA_FCOE_MODE);
		rc = -EIO;
		kfree(vpd);
		goto out_free_mbox;
	}

	/*
	 * Continue initialization with default values even if driver failed
	 * to read FCoE param config regions, only read parameters if the
	 * board is FCoE
	 */
	if (phba->hba_flag & HBA_FCOE_MODE &&
	    lpfc_sli4_read_fcoe_params(phba))
		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_INIT,
			"2570 Failed to read FCoE parameters\n");

	/*
	 * Retrieve sli4 device physical port name, failure of doing it
	 * is considered as non-fatal.
	 */
	rc = lpfc_sli4_retrieve_pport_name(phba);
	if (!rc)
		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
				"3080 Successful retrieving SLI4 device "
				"physical port name: %s.\n", phba->Port);

	rc = lpfc_sli4_get_ctl_attr(phba);
	if (!rc)
		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
				"8351 Successful retrieving SLI4 device "
				"CTL ATTR\n");

	/*
	 * Evaluate the read rev and vpd data. Populate the driver
	 * state with the results. If this routine fails, the failure
	 * is not fatal as the driver will use generic values.
	 */
	rc = lpfc_parse_vpd(phba, vpd, vpd_size);
	if (unlikely(!rc)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"0377 Error %d parsing vpd. "
				"Using defaults.\n", rc);
		rc = 0;
	}
	kfree(vpd);

	/* Save information as VPD data */
	phba->vpd.rev.biuRev = mqe->un.read_rev.first_hw_rev;
	phba->vpd.rev.smRev = mqe->un.read_rev.second_hw_rev;

	/*
	 * This is because first G7 ASIC doesn't support the standard
	 * 0x5a NVME cmd descriptor type/subtype
	 */
	if ((bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
			LPFC_SLI_INTF_IF_TYPE_6) &&
	    (phba->vpd.rev.biuRev == LPFC_G7_ASIC_1) &&
	    (phba->vpd.rev.smRev == 0) &&
	    (phba->cfg_nvme_embed_cmd == 1))
		phba->cfg_nvme_embed_cmd = 0;

	phba->vpd.rev.endecRev = mqe->un.read_rev.third_hw_rev;
	phba->vpd.rev.fcphHigh = bf_get(lpfc_mbx_rd_rev_fcph_high,
					 &mqe->un.read_rev);
	phba->vpd.rev.fcphLow = bf_get(lpfc_mbx_rd_rev_fcph_low,
				       &mqe->un.read_rev);
	phba->vpd.rev.feaLevelHigh = bf_get(lpfc_mbx_rd_rev_ftr_lvl_high,
					    &mqe->un.read_rev);
	phba->vpd.rev.feaLevelLow = bf_get(lpfc_mbx_rd_rev_ftr_lvl_low,
					   &mqe->un.read_rev);
	phba->vpd.rev.sli1FwRev = mqe->un.read_rev.fw_id_rev;
	memcpy(phba->vpd.rev.sli1FwName, mqe->un.read_rev.fw_name, 16);
	phba->vpd.rev.sli2FwRev = mqe->un.read_rev.ulp_fw_id_rev;
	memcpy(phba->vpd.rev.sli2FwName, mqe->un.read_rev.ulp_fw_name, 16);
	phba->vpd.rev.opFwRev = mqe->un.read_rev.fw_id_rev;
	memcpy(phba->vpd.rev.opFwName, mqe->un.read_rev.fw_name, 16);
	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
			"(%d):0380 READ_REV Status x%x "
			"fw_rev:%s fcphHi:%x fcphLo:%x flHi:%x flLo:%x\n",
			mboxq->vport ? mboxq->vport->vpi : 0,
			bf_get(lpfc_mqe_status, mqe),
			phba->vpd.rev.opFwName,
			phba->vpd.rev.fcphHigh, phba->vpd.rev.fcphLow,
			phba->vpd.rev.feaLevelHigh, phba->vpd.rev.feaLevelLow);

	/* Reset the DFT_LUN_Q_DEPTH to (max xri >> 3)  */
	rc = (phba->sli4_hba.max_cfg_param.max_xri >> 3);
	if (phba->pport->cfg_lun_queue_depth > rc) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
				"3362 LUN queue depth changed from %d to %d\n",
				phba->pport->cfg_lun_queue_depth, rc);
		phba->pport->cfg_lun_queue_depth = rc;
	}

	if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
	    LPFC_SLI_INTF_IF_TYPE_0) {
		lpfc_set_features(phba, mboxq, LPFC_SET_UE_RECOVERY);
		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
		if (rc == MBX_SUCCESS) {
			phba->hba_flag |= HBA_RECOVERABLE_UE;
			/* Set 1Sec interval to detect UE */
			phba->eratt_poll_interval = 1;
			phba->sli4_hba.ue_to_sr = bf_get(
					lpfc_mbx_set_feature_UESR,
					&mboxq->u.mqe.un.set_feature);
			phba->sli4_hba.ue_to_rp = bf_get(
					lpfc_mbx_set_feature_UERP,
					&mboxq->u.mqe.un.set_feature);
		}
	}

	if (phba->cfg_enable_mds_diags && phba->mds_diags_support) {
		/* Enable MDS Diagnostics only if the SLI Port supports it */
		lpfc_set_features(phba, mboxq, LPFC_SET_MDS_DIAGS);
		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
		if (rc != MBX_SUCCESS)
			phba->mds_diags_support = 0;
	}

	/*
	 * Discover the port's supported feature set and match it against the
	 * hosts requests.
	 */
	lpfc_request_features(phba, mboxq);
	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
	if (unlikely(rc)) {
		rc = -EIO;
		goto out_free_mbox;
	}

	/*
	 * The port must support FCP initiator mode as this is the
	 * only mode running in the host.
	 */
	if (!(bf_get(lpfc_mbx_rq_ftr_rsp_fcpi, &mqe->un.req_ftrs))) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
				"0378 No support for fcpi mode.\n");
		ftr_rsp++;
	}

	/* Performance Hints are ONLY for FCoE */
	if (phba->hba_flag & HBA_FCOE_MODE) {
		if (bf_get(lpfc_mbx_rq_ftr_rsp_perfh, &mqe->un.req_ftrs))
			phba->sli3_options |= LPFC_SLI4_PERFH_ENABLED;
		else
			phba->sli3_options &= ~LPFC_SLI4_PERFH_ENABLED;
	}

	/*
	 * If the port cannot support the host's requested features
	 * then turn off the global config parameters to disable the
	 * feature in the driver.  This is not a fatal error.
	 */
	if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
		if (!(bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs))) {
			phba->cfg_enable_bg = 0;
			phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
			ftr_rsp++;
		}
	}

	if (phba->max_vpi && phba->cfg_enable_npiv &&
	    !(bf_get(lpfc_mbx_rq_ftr_rsp_npiv, &mqe->un.req_ftrs)))
		ftr_rsp++;

	if (ftr_rsp) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
				"0379 Feature Mismatch Data: x%08x %08x "
				"x%x x%x x%x\n", mqe->un.req_ftrs.word2,
				mqe->un.req_ftrs.word3, phba->cfg_enable_bg,
				phba->cfg_enable_npiv, phba->max_vpi);
		if (!(bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs)))
			phba->cfg_enable_bg = 0;
		if (!(bf_get(lpfc_mbx_rq_ftr_rsp_npiv, &mqe->un.req_ftrs)))
			phba->cfg_enable_npiv = 0;
	}

	/* These SLI3 features are assumed in SLI4 */
	spin_lock_irq(&phba->hbalock);
	phba->sli3_options |= (LPFC_SLI3_NPIV_ENABLED | LPFC_SLI3_HBQ_ENABLED);
	spin_unlock_irq(&phba->hbalock);

	/*
	 * Allocate all resources (xri,rpi,vpi,vfi) now.  Subsequent
	 * calls depends on these resources to complete port setup.
	 */
	rc = lpfc_sli4_alloc_resource_identifiers(phba);
	if (rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"2920 Failed to alloc Resource IDs "
				"rc = x%x\n", rc);
		goto out_free_mbox;
	}

	lpfc_set_host_data(phba, mboxq);

	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
	if (rc) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
				"2134 Failed to set host os driver version %x",
				rc);
	}

	/* Read the port's service parameters. */
	rc = lpfc_read_sparam(phba, mboxq, vport->vpi);
	if (rc) {
		phba->link_state = LPFC_HBA_ERROR;
		rc = -ENOMEM;
		goto out_free_mbox;
	}

	mboxq->vport = vport;
	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
	mp = (struct lpfc_dmabuf *)mboxq->ctx_buf;
	if (rc == MBX_SUCCESS) {
		memcpy(&vport->fc_sparam, mp->virt, sizeof(struct serv_parm));
		rc = 0;
	}

	/*
	 * This memory was allocated by the lpfc_read_sparam routine. Release
	 * it to the mbuf pool.
	 */
	lpfc_mbuf_free(phba, mp->virt, mp->phys);
	kfree(mp);
	mboxq->ctx_buf = NULL;
	if (unlikely(rc)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"0382 READ_SPARAM command failed "
				"status %d, mbxStatus x%x\n",
				rc, bf_get(lpfc_mqe_status, mqe));
		phba->link_state = LPFC_HBA_ERROR;
		rc = -EIO;
		goto out_free_mbox;
	}

	lpfc_update_vport_wwn(vport);

	/* Update the fc_host data structures with new wwn. */
	fc_host_node_name(shost) = wwn_to_u64(vport->fc_nodename.u.wwn);
	fc_host_port_name(shost) = wwn_to_u64(vport->fc_portname.u.wwn);

	/* Create all the SLI4 queues */
	rc = lpfc_sli4_queue_create(phba);
	if (rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"3089 Failed to allocate queues\n");
		rc = -ENODEV;
		goto out_free_mbox;
	}
	/* Set up all the queues to the device */
	rc = lpfc_sli4_queue_setup(phba);
	if (unlikely(rc)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"0381 Error %d during queue setup.\n ", rc);
		goto out_stop_timers;
	}
	/* Initialize the driver internal SLI layer lists. */
	lpfc_sli4_setup(phba);
	lpfc_sli4_queue_init(phba);

	/* update host els xri-sgl sizes and mappings */
	rc = lpfc_sli4_els_sgl_update(phba);
	if (unlikely(rc)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"1400 Failed to update xri-sgl size and "
				"mapping: %d\n", rc);
		goto out_destroy_queue;
	}

	/* register the els sgl pool to the port */
	rc = lpfc_sli4_repost_sgl_list(phba, &phba->sli4_hba.lpfc_els_sgl_list,
				       phba->sli4_hba.els_xri_cnt);
	if (unlikely(rc < 0)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"0582 Error %d during els sgl post "
				"operation\n", rc);
		rc = -ENODEV;
		goto out_destroy_queue;
	}
	phba->sli4_hba.els_xri_cnt = rc;

	if (phba->nvmet_support) {
		/* update host nvmet xri-sgl sizes and mappings */
		rc = lpfc_sli4_nvmet_sgl_update(phba);
		if (unlikely(rc)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"6308 Failed to update nvmet-sgl size "
					"and mapping: %d\n", rc);
			goto out_destroy_queue;
		}

		/* register the nvmet sgl pool to the port */
		rc = lpfc_sli4_repost_sgl_list(
			phba,
			&phba->sli4_hba.lpfc_nvmet_sgl_list,
			phba->sli4_hba.nvmet_xri_cnt);
		if (unlikely(rc < 0)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"3117 Error %d during nvmet "
					"sgl post\n", rc);
			rc = -ENODEV;
			goto out_destroy_queue;
		}
		phba->sli4_hba.nvmet_xri_cnt = rc;

		cnt = phba->cfg_iocb_cnt * 1024;
		/* We need 1 iocbq for every SGL, for IO processing */
		cnt += phba->sli4_hba.nvmet_xri_cnt;
	} else {
		/* update host common xri-sgl sizes and mappings */
		rc = lpfc_sli4_io_sgl_update(phba);
		if (unlikely(rc)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"6082 Failed to update nvme-sgl size "
					"and mapping: %d\n", rc);
			goto out_destroy_queue;
		}

		/* register the allocated common sgl pool to the port */
		rc = lpfc_sli4_repost_io_sgl_list(phba);
		if (unlikely(rc)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"6116 Error %d during nvme sgl post "
					"operation\n", rc);
			/* Some NVME buffers were moved to abort nvme list */
			/* A pci function reset will repost them */
			rc = -ENODEV;
			goto out_destroy_queue;
		}
		cnt = phba->cfg_iocb_cnt * 1024;
	}

	if (!phba->sli.iocbq_lookup) {
		/* Initialize and populate the iocb list per host */
		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
				"2821 initialize iocb list %d total %d\n",
				phba->cfg_iocb_cnt, cnt);
		rc = lpfc_init_iocb_list(phba, cnt);
		if (rc) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"1413 Failed to init iocb list.\n");
			goto out_destroy_queue;
		}
	}

	if (phba->nvmet_support)
		lpfc_nvmet_create_targetport(phba);

	if (phba->nvmet_support && phba->cfg_nvmet_mrq) {
		/* Post initial buffers to all RQs created */
		for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
			rqbp = phba->sli4_hba.nvmet_mrq_hdr[i]->rqbp;
			INIT_LIST_HEAD(&rqbp->rqb_buffer_list);
			rqbp->rqb_alloc_buffer = lpfc_sli4_nvmet_alloc;
			rqbp->rqb_free_buffer = lpfc_sli4_nvmet_free;
			rqbp->entry_count = LPFC_NVMET_RQE_DEF_COUNT;
			rqbp->buffer_count = 0;

			lpfc_post_rq_buffer(
				phba, phba->sli4_hba.nvmet_mrq_hdr[i],
				phba->sli4_hba.nvmet_mrq_data[i],
				phba->cfg_nvmet_mrq_post, i);
		}
	}

	/* Post the rpi header region to the device. */
	rc = lpfc_sli4_post_all_rpi_hdrs(phba);
	if (unlikely(rc)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"0393 Error %d during rpi post operation\n",
				rc);
		rc = -ENODEV;
		goto out_destroy_queue;
	}
	lpfc_sli4_node_prep(phba);

	if (!(phba->hba_flag & HBA_FCOE_MODE)) {
		if ((phba->nvmet_support == 0) || (phba->cfg_nvmet_mrq == 1)) {
			/*
			 * The FC Port needs to register FCFI (index 0)
			 */
			lpfc_reg_fcfi(phba, mboxq);
			mboxq->vport = phba->pport;
			rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
			if (rc != MBX_SUCCESS)
				goto out_unset_queue;
			rc = 0;
			phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_fcfi,
						&mboxq->u.mqe.un.reg_fcfi);
		} else {
			/* We are a NVME Target mode with MRQ > 1 */

			/* First register the FCFI */
			lpfc_reg_fcfi_mrq(phba, mboxq, 0);
			mboxq->vport = phba->pport;
			rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
			if (rc != MBX_SUCCESS)
				goto out_unset_queue;
			rc = 0;
			phba->fcf.fcfi = bf_get(lpfc_reg_fcfi_mrq_fcfi,
						&mboxq->u.mqe.un.reg_fcfi_mrq);

			/* Next register the MRQs */
			lpfc_reg_fcfi_mrq(phba, mboxq, 1);
			mboxq->vport = phba->pport;
			rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
			if (rc != MBX_SUCCESS)
				goto out_unset_queue;
			rc = 0;
		}
		/* Check if the port is configured to be disabled */
		lpfc_sli_read_link_ste(phba);
	}

	/* Don't post more new bufs if repost already recovered
	 * the nvme sgls.
	 */
	if (phba->nvmet_support == 0) {
		if (phba->sli4_hba.io_xri_cnt == 0) {
			len = lpfc_new_io_buf(
					      phba, phba->sli4_hba.io_xri_max);
			if (len == 0) {
				rc = -ENOMEM;
				goto out_unset_queue;
			}

			if (phba->cfg_xri_rebalancing)
				lpfc_create_multixri_pools(phba);
		}
	} else {
		phba->cfg_xri_rebalancing = 0;
	}

	/* Allow asynchronous mailbox command to go through */
	spin_lock_irq(&phba->hbalock);
	phba->sli.sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
	spin_unlock_irq(&phba->hbalock);

	/* Post receive buffers to the device */
	lpfc_sli4_rb_setup(phba);

	/* Reset HBA FCF states after HBA reset */
	phba->fcf.fcf_flag = 0;
	phba->fcf.current_rec.flag = 0;

	/* Start the ELS watchdog timer */
	mod_timer(&vport->els_tmofunc,
		  jiffies + msecs_to_jiffies(1000 * (phba->fc_ratov * 2)));

	/* Start heart beat timer */
	mod_timer(&phba->hb_tmofunc,
		  jiffies + msecs_to_jiffies(1000 * LPFC_HB_MBOX_INTERVAL));
	phba->hb_outstanding = 0;
	phba->last_completion_time = jiffies;

	/* start eq_delay heartbeat */
	if (phba->cfg_auto_imax)
		queue_delayed_work(phba->wq, &phba->eq_delay_work,
				   msecs_to_jiffies(LPFC_EQ_DELAY_MSECS));

	/* Start error attention (ERATT) polling timer */
	mod_timer(&phba->eratt_poll,
		  jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval));

	/* Enable PCIe device Advanced Error Reporting (AER) if configured */
	if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) {
		rc = pci_enable_pcie_error_reporting(phba->pcidev);
		if (!rc) {
			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
					"2829 This device supports "
					"Advanced Error Reporting (AER)\n");
			spin_lock_irq(&phba->hbalock);
			phba->hba_flag |= HBA_AER_ENABLED;
			spin_unlock_irq(&phba->hbalock);
		} else {
			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
					"2830 This device does not support "
					"Advanced Error Reporting (AER)\n");
			phba->cfg_aer_support = 0;
		}
		rc = 0;
	}

	/*
	 * The port is ready, set the host's link state to LINK_DOWN
	 * in preparation for link interrupts.
	 */
	spin_lock_irq(&phba->hbalock);
	phba->link_state = LPFC_LINK_DOWN;

	/* Check if physical ports are trunked */
	if (bf_get(lpfc_conf_trunk_port0, &phba->sli4_hba))
		phba->trunk_link.link0.state = LPFC_LINK_DOWN;
	if (bf_get(lpfc_conf_trunk_port1, &phba->sli4_hba))
		phba->trunk_link.link1.state = LPFC_LINK_DOWN;
	if (bf_get(lpfc_conf_trunk_port2, &phba->sli4_hba))
		phba->trunk_link.link2.state = LPFC_LINK_DOWN;
	if (bf_get(lpfc_conf_trunk_port3, &phba->sli4_hba))
		phba->trunk_link.link3.state = LPFC_LINK_DOWN;
	spin_unlock_irq(&phba->hbalock);

	/* Arm the CQs and then EQs on device */
	lpfc_sli4_arm_cqeq_intr(phba);

	/* Indicate device interrupt mode */
	phba->sli4_hba.intr_enable = 1;

	if (!(phba->hba_flag & HBA_FCOE_MODE) &&
	    (phba->hba_flag & LINK_DISABLED)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI,
				"3103 Adapter Link is disabled.\n");
		lpfc_down_link(phba, mboxq);
		rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
		if (rc != MBX_SUCCESS) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI,
					"3104 Adapter failed to issue "
					"DOWN_LINK mbox cmd, rc:x%x\n", rc);
			goto out_io_buff_free;
		}
	} else if (phba->cfg_suppress_link_up == LPFC_INITIALIZE_LINK) {
		/* don't perform init_link on SLI4 FC port loopback test */
		if (!(phba->link_flag & LS_LOOPBACK_MODE)) {
			rc = phba->lpfc_hba_init_link(phba, MBX_NOWAIT);
			if (rc)
				goto out_io_buff_free;
		}
	}
	mempool_free(mboxq, phba->mbox_mem_pool);
	return rc;
out_io_buff_free:
	/* Free allocated IO Buffers */
	lpfc_io_free(phba);
out_unset_queue:
	/* Unset all the queues set up in this routine when error out */
	lpfc_sli4_queue_unset(phba);
out_destroy_queue:
	lpfc_free_iocb_list(phba);
	lpfc_sli4_queue_destroy(phba);
out_stop_timers:
	lpfc_stop_hba_timers(phba);
out_free_mbox:
	mempool_free(mboxq, phba->mbox_mem_pool);
	return rc;
}

/**
 * lpfc_mbox_timeout - Timeout call back function for mbox timer
 * @ptr: context object - pointer to hba structure.
 *
 * This is the callback function for mailbox timer. The mailbox
 * timer is armed when a new mailbox command is issued and the timer
 * is deleted when the mailbox complete. The function is called by
 * the kernel timer code when a mailbox does not complete within
 * expected time. This function wakes up the worker thread to
 * process the mailbox timeout and returns. All the processing is
 * done by the worker thread function lpfc_mbox_timeout_handler.
 **/
void
lpfc_mbox_timeout(struct timer_list *t)
{
	struct lpfc_hba  *phba = from_timer(phba, t, sli.mbox_tmo);
	unsigned long iflag;
	uint32_t tmo_posted;

	spin_lock_irqsave(&phba->pport->work_port_lock, iflag);
	tmo_posted = phba->pport->work_port_events & WORKER_MBOX_TMO;
	if (!tmo_posted)
		phba->pport->work_port_events |= WORKER_MBOX_TMO;
	spin_unlock_irqrestore(&phba->pport->work_port_lock, iflag);

	if (!tmo_posted)
		lpfc_worker_wake_up(phba);
	return;
}

/**
 * lpfc_sli4_mbox_completions_pending - check to see if any mailbox completions
 *                                    are pending
 * @phba: Pointer to HBA context object.
 *
 * This function checks if any mailbox completions are present on the mailbox
 * completion queue.
 **/
static bool
lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
{

	uint32_t idx;
	struct lpfc_queue *mcq;
	struct lpfc_mcqe *mcqe;
	bool pending_completions = false;
	uint8_t	qe_valid;

	if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4))
		return false;

	/* Check for completions on mailbox completion queue */

	mcq = phba->sli4_hba.mbx_cq;
	idx = mcq->hba_index;
	qe_valid = mcq->qe_valid;
	while (bf_get_le32(lpfc_cqe_valid,
	       (struct lpfc_cqe *)lpfc_sli4_qe(mcq, idx)) == qe_valid) {
		mcqe = (struct lpfc_mcqe *)(lpfc_sli4_qe(mcq, idx));
		if (bf_get_le32(lpfc_trailer_completed, mcqe) &&
		    (!bf_get_le32(lpfc_trailer_async, mcqe))) {
			pending_completions = true;
			break;
		}
		idx = (idx + 1) % mcq->entry_count;
		if (mcq->hba_index == idx)
			break;

		/* if the index wrapped around, toggle the valid bit */
		if (phba->sli4_hba.pc_sli4_params.cqav && !idx)
			qe_valid = (qe_valid) ? 0 : 1;
	}
	return pending_completions;

}

/**
 * lpfc_sli4_process_missed_mbox_completions - process mbox completions
 *					      that were missed.
 * @phba: Pointer to HBA context object.
 *
 * For sli4, it is possible to miss an interrupt. As such mbox completions
 * maybe missed causing erroneous mailbox timeouts to occur. This function
 * checks to see if mbox completions are on the mailbox completion queue
 * and will process all the completions associated with the eq for the
 * mailbox completion queue.
 **/
bool
lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba)
{
	struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
	uint32_t eqidx;
	struct lpfc_queue *fpeq = NULL;
	bool mbox_pending;

	if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4))
		return false;

	/* Find the eq associated with the mcq */

	if (sli4_hba->hdwq)
		for (eqidx = 0; eqidx < phba->cfg_irq_chann; eqidx++)
			if (sli4_hba->hdwq[eqidx].hba_eq->queue_id ==
			    sli4_hba->mbx_cq->assoc_qid) {
				fpeq = sli4_hba->hdwq[eqidx].hba_eq;
				break;
			}
	if (!fpeq)
		return false;

	/* Turn off interrupts from this EQ */

	sli4_hba->sli4_eq_clr_intr(fpeq);

	/* Check to see if a mbox completion is pending */

	mbox_pending = lpfc_sli4_mbox_completions_pending(phba);

	/*
	 * If a mbox completion is pending, process all the events on EQ
	 * associated with the mbox completion queue (this could include
	 * mailbox commands, async events, els commands, receive queue data
	 * and fcp commands)
	 */

	if (mbox_pending)
		/* process and rearm the EQ */
		lpfc_sli4_process_eq(phba, fpeq);
	else
		/* Always clear and re-arm the EQ */
		sli4_hba->sli4_write_eq_db(phba, fpeq, 0, LPFC_QUEUE_REARM);

	return mbox_pending;

}

/**
 * lpfc_mbox_timeout_handler - Worker thread function to handle mailbox timeout
 * @phba: Pointer to HBA context object.
 *
 * This function is called from worker thread when a mailbox command times out.
 * The caller is not required to hold any locks. This function will reset the
 * HBA and recover all the pending commands.
 **/
void
lpfc_mbox_timeout_handler(struct lpfc_hba *phba)
{
	LPFC_MBOXQ_t *pmbox = phba->sli.mbox_active;
	MAILBOX_t *mb = NULL;

	struct lpfc_sli *psli = &phba->sli;

	/* If the mailbox completed, process the completion and return */
	if (lpfc_sli4_process_missed_mbox_completions(phba))
		return;

	if (pmbox != NULL)
		mb = &pmbox->u.mb;
	/* Check the pmbox pointer first.  There is a race condition
	 * between the mbox timeout handler getting executed in the
	 * worklist and the mailbox actually completing. When this
	 * race condition occurs, the mbox_active will be NULL.
	 */
	spin_lock_irq(&phba->hbalock);
	if (pmbox == NULL) {
		lpfc_printf_log(phba, KERN_WARNING,
				LOG_MBOX | LOG_SLI,
				"0353 Active Mailbox cleared - mailbox timeout "
				"exiting\n");
		spin_unlock_irq(&phba->hbalock);
		return;
	}

	/* Mbox cmd <mbxCommand> timeout */
	lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
			"0310 Mailbox command x%x timeout Data: x%x x%x x%p\n",
			mb->mbxCommand,
			phba->pport->port_state,
			phba->sli.sli_flag,
			phba->sli.mbox_active);
	spin_unlock_irq(&phba->hbalock);

	/* Setting state unknown so lpfc_sli_abort_iocb_ring
	 * would get IOCB_ERROR from lpfc_sli_issue_iocb, allowing
	 * it to fail all outstanding SCSI IO.
	 */
	spin_lock_irq(&phba->pport->work_port_lock);
	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
	spin_unlock_irq(&phba->pport->work_port_lock);
	spin_lock_irq(&phba->hbalock);
	phba->link_state = LPFC_LINK_UNKNOWN;
	psli->sli_flag &= ~LPFC_SLI_ACTIVE;
	spin_unlock_irq(&phba->hbalock);

	lpfc_sli_abort_fcp_rings(phba);

	lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
			"0345 Resetting board due to mailbox timeout\n");

	/* Reset the HBA device */
	lpfc_reset_hba(phba);
}

/**
 * lpfc_sli_issue_mbox_s3 - Issue an SLI3 mailbox command to firmware
 * @phba: Pointer to HBA context object.
 * @pmbox: Pointer to mailbox object.
 * @flag: Flag indicating how the mailbox need to be processed.
 *
 * This function is called by discovery code and HBA management code
 * to submit a mailbox command to firmware with SLI-3 interface spec. This
 * function gets the hbalock to protect the data structures.
 * The mailbox command can be submitted in polling mode, in which case
 * this function will wait in a polling loop for the completion of the
 * mailbox.
 * If the mailbox is submitted in no_wait mode (not polling) the
 * function will submit the command and returns immediately without waiting
 * for the mailbox completion. The no_wait is supported only when HBA
 * is in SLI2/SLI3 mode - interrupts are enabled.
 * The SLI interface allows only one mailbox pending at a time. If the
 * mailbox is issued in polling mode and there is already a mailbox
 * pending, then the function will return an error. If the mailbox is issued
 * in NO_WAIT mode and there is a mailbox pending already, the function
 * will return MBX_BUSY after queuing the mailbox into mailbox queue.
 * The sli layer owns the mailbox object until the completion of mailbox
 * command if this function return MBX_BUSY or MBX_SUCCESS. For all other
 * return codes the caller owns the mailbox command after the return of
 * the function.
 **/
static int
lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox,
		       uint32_t flag)
{
	MAILBOX_t *mbx;
	struct lpfc_sli *psli = &phba->sli;
	uint32_t status, evtctr;
	uint32_t ha_copy, hc_copy;
	int i;
	unsigned long timeout;
	unsigned long drvr_flag = 0;
	uint32_t word0, ldata;
	void __iomem *to_slim;
	int processing_queue = 0;

	spin_lock_irqsave(&phba->hbalock, drvr_flag);
	if (!pmbox) {
		phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
		/* processing mbox queue from intr_handler */
		if (unlikely(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
			return MBX_SUCCESS;
		}
		processing_queue = 1;
		pmbox = lpfc_mbox_get(phba);
		if (!pmbox) {
			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
			return MBX_SUCCESS;
		}
	}

	if (pmbox->mbox_cmpl && pmbox->mbox_cmpl != lpfc_sli_def_mbox_cmpl &&
		pmbox->mbox_cmpl != lpfc_sli_wake_mbox_wait) {
		if(!pmbox->vport) {
			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
			lpfc_printf_log(phba, KERN_ERR,
					LOG_MBOX | LOG_VPORT,
					"1806 Mbox x%x failed. No vport\n",
					pmbox->u.mb.mbxCommand);
			dump_stack();
			goto out_not_finished;
		}
	}

	/* If the PCI channel is in offline state, do not post mbox. */
	if (unlikely(pci_channel_offline(phba->pcidev))) {
		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
		goto out_not_finished;
	}

	/* If HBA has a deferred error attention, fail the iocb. */
	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
		goto out_not_finished;
	}

	psli = &phba->sli;

	mbx = &pmbox->u.mb;
	status = MBX_SUCCESS;

	if (phba->link_state == LPFC_HBA_ERROR) {
		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);

		/* Mbox command <mbxCommand> cannot issue */
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"(%d):0311 Mailbox command x%x cannot "
				"issue Data: x%x x%x\n",
				pmbox->vport ? pmbox->vport->vpi : 0,
				pmbox->u.mb.mbxCommand, psli->sli_flag, flag);
		goto out_not_finished;
	}

	if (mbx->mbxCommand != MBX_KILL_BOARD && flag & MBX_NOWAIT) {
		if (lpfc_readl(phba->HCregaddr, &hc_copy) ||
			!(hc_copy & HC_MBINT_ENA)) {
			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"(%d):2528 Mailbox command x%x cannot "
				"issue Data: x%x x%x\n",
				pmbox->vport ? pmbox->vport->vpi : 0,
				pmbox->u.mb.mbxCommand, psli->sli_flag, flag);
			goto out_not_finished;
		}
	}

	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
		/* Polling for a mbox command when another one is already active
		 * is not allowed in SLI. Also, the driver must have established
		 * SLI2 mode to queue and process multiple mbox commands.
		 */

		if (flag & MBX_POLL) {
			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);

			/* Mbox command <mbxCommand> cannot issue */
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"(%d):2529 Mailbox command x%x "
					"cannot issue Data: x%x x%x\n",
					pmbox->vport ? pmbox->vport->vpi : 0,
					pmbox->u.mb.mbxCommand,
					psli->sli_flag, flag);
			goto out_not_finished;
		}

		if (!(psli->sli_flag & LPFC_SLI_ACTIVE)) {
			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
			/* Mbox command <mbxCommand> cannot issue */
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"(%d):2530 Mailbox command x%x "
					"cannot issue Data: x%x x%x\n",
					pmbox->vport ? pmbox->vport->vpi : 0,
					pmbox->u.mb.mbxCommand,
					psli->sli_flag, flag);
			goto out_not_finished;
		}

		/* Another mailbox command is still being processed, queue this
		 * command to be processed later.
		 */
		lpfc_mbox_put(phba, pmbox);

		/* Mbox cmd issue - BUSY */
		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
				"(%d):0308 Mbox cmd issue - BUSY Data: "
				"x%x x%x x%x x%x\n",
				pmbox->vport ? pmbox->vport->vpi : 0xffffff,
				mbx->mbxCommand,
				phba->pport ? phba->pport->port_state : 0xff,
				psli->sli_flag, flag);

		psli->slistat.mbox_busy++;
		spin_unlock_irqrestore(&phba->hbalock, drvr_flag);

		if (pmbox->vport) {
			lpfc_debugfs_disc_trc(pmbox->vport,
				LPFC_DISC_TRC_MBOX_VPORT,
				"MBOX Bsy vport:  cmd:x%x mb:x%x x%x",
				(uint32_t)mbx->mbxCommand,
				mbx->un.varWords[0], mbx->un.varWords[1]);
		}
		else {
			lpfc_debugfs_disc_trc(phba->pport,
				LPFC_DISC_TRC_MBOX,
				"MBOX Bsy:        cmd:x%x mb:x%x x%x",
				(uint32_t)mbx->mbxCommand,
				mbx->un.varWords[0], mbx->un.varWords[1]);
		}

		return MBX_BUSY;
	}

	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;

	/* If we are not polling, we MUST be in SLI2 mode */
	if (flag != MBX_POLL) {
		if (!(psli->sli_flag & LPFC_SLI_ACTIVE) &&
		    (mbx->mbxCommand != MBX_KILL_BOARD)) {
			psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
			spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
			/* Mbox command <mbxCommand> cannot issue */
			lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
					"(%d):2531 Mailbox command x%x "
					"cannot issue Data: x%x x%x\n",
					pmbox->vport ? pmbox->vport->vpi : 0,
					pmbox->u.mb.mbxCommand,
					psli->sli_flag, flag);
			goto out_not_finished;
		}
		/* timeout active mbox command */
		timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox) *
					   1000);
		mod_timer(&psli->mbox_tmo, jiffies + timeout);
	}

	/* Mailbox cmd <cmd> issue */
	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
			"(%d):0309 Mailbox cmd x%x issue Data: x%x x%x "
			"x%x\n",
			pmbox->vport ? pmbox->vport->vpi : 0,
			mbx->mbxCommand,
			phba->pport ? phba->pport->port_state : 0xff,
			psli->sli_flag, flag);

	if (mbx->mbxCommand != MBX_HEARTBEAT) {
		if (pmbox->vport) {
			lpfc_debugfs_disc_trc(pmbox->vport,
				LPFC_DISC_TRC_MBOX_VPORT,
				"MBOX Send vport: cmd:x%x mb:x%x x%x",
				(uint32_t)mbx->mbxCommand,
				mbx->un.varWords[0], mbx->un.varWords[1]);
		}
		else {
			lpfc_debugfs_disc_trc(phba->pport,
				LPFC_DISC_TRC_MBOX,
				"MBOX Send:       cmd:x%x mb:x%x x%x",
				(uint32_t)mbx->mbxCommand,
				mbx->un.varWords[0], mbx->un.varWords[1]);
		}
	}

	psli->slistat.mbox_cmd++;
	evtctr = psli->slistat.mbox_event;

	/* next set own bit for the adapter and copy over command word */
	mbx->mbxOwner = OWN_CHIP;

	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
		/* Populate mbox extension offset word. */
		if (pmbox->in_ext_byte_len || pmbox->out_ext_byte_len) {
			*(((uint32_t *)mbx) + pmbox->mbox_offset_word)
				= (uint8_t *)phba->mbox_ext
				  - (uint8_t *)phba->mbox;
		}

		/* Copy the mailbox extension data */
		if (pmbox->in_ext_byte_len && pmbox->ctx_buf) {
			lpfc_sli_pcimem_bcopy(pmbox->ctx_buf,
					      (uint8_t *)phba->mbox_ext,
					      pmbox->in_ext_byte_len);
		}
		/* Copy command data to host SLIM area */
		lpfc_sli_pcimem_bcopy(mbx, phba->mbox, MAILBOX_CMD_SIZE);
	} else {
		/* Populate mbox extension offset word. */
		if (pmbox->in_ext_byte_len || pmbox->out_ext_byte_len)
			*(((uint32_t *)mbx) + pmbox->mbox_offset_word)
				= MAILBOX_HBA_EXT_OFFSET;

		/* Copy the mailbox extension data */
		if (pmbox->in_ext_byte_len && pmbox->ctx_buf)
			lpfc_memcpy_to_slim(phba->MBslimaddr +
				MAILBOX_HBA_EXT_OFFSET,
				pmbox->ctx_buf, pmbox->in_ext_byte_len);

		if (mbx->mbxCommand == MBX_CONFIG_PORT)
			/* copy command data into host mbox for cmpl */
			lpfc_sli_pcimem_bcopy(mbx, phba->mbox,
					      MAILBOX_CMD_SIZE);

		/* First copy mbox command data to HBA SLIM, skip past first
		   word */
		to_slim = phba->MBslimaddr + sizeof (uint32_t);
		lpfc_memcpy_to_slim(to_slim, &mbx->un.varWords[0],
			    MAILBOX_CMD_SIZE - sizeof (uint32_t));

		/* Next copy over first word, with mbxOwner set */
		ldata = *((uint32_t *)mbx);
		to_slim = phba->MBslimaddr;
		writel(ldata, to_slim);
		readl(to_slim); /* flush */

		if (mbx->mbxCommand == MBX_CONFIG_PORT)
			/* switch over to host mailbox */
			psli->sli_flag |= LPFC_SLI_ACTIVE;
	}

	wmb();

	switch (flag) {
	case MBX_NOWAIT:
		/* Set up reference to mailbox command */
		psli->mbox_active = pmbox;
		/* Interrupt board to do it */
		writel(CA_MBATT, phba->CAregaddr);
		readl(phba->CAregaddr); /* flush */
		/* Don't wait for it to finish, just return */
		break;

	case MBX_POLL:
		/* Set up null reference to mailbox command */
		psli->mbox_active = NULL;
		/* Interrupt board to do it */
		writel(CA_MBATT, phba->CAregaddr);
		readl(phba->CAregaddr); /* flush */

		if (psli->sli_flag & LPFC_SLI_ACTIVE) {
			/* First read mbox status word */
			word0 = *((uint32_t *)phba->mbox);
			word0 = le32_to_cpu(word0);
		} else {
			/* First read mbox status word */
			if (lpfc_readl(phba->MBslimaddr, &word0)) {
				spin_unlock_irqrestore(&phba->hbalock,
						       drvr_flag);
				goto out_not_finished;
			}
		}

		/* Read the HBA Host Attention Register */
		if (lpfc_readl(phba->HAregaddr, &ha_copy)) {
			spin_unlock_irqrestore(&phba->hbalock,
						       drvr_flag);
			goto out_not_finished;
		}
		timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox) *
							1000) + jiffies;
		i = 0;
		/* Wait for command to complete */
		while (((word0 & OWN_CHIP) == OWN_CHIP) ||
		       (!(ha_copy & HA_MBATT) &&
			(phba->link_state > LPFC_WARM_START))) {
			if (time_after(jiffies, timeout)) {
				psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
				spin_unlock_irqrestore(&phba->hbalock,
						       drvr_flag);
				goto out_not_finished;
			}

			/* Check if we took a mbox interrupt while we were
			   polling */
			if (((word0 & OWN_CHIP) != OWN_CHIP)
			    && (evtctr != psli->slistat.mbox_event))
				break;

			if (i++ > 10) {
				spin_unlock_irqrestore(&phba->hbalock,
						       drvr_flag);
				msleep(1);
				spin_lock_irqsave(&phba->hbalock, drvr_flag);
			}

			if (psli->sli_flag & LPFC_SLI_ACTIVE) {
				/* First copy command data */
				word0 = *((uint32_t *)phba->mbox);
				word0 = le32_to_cpu(word0);
				if (mbx->mbxCommand == MBX_CONFIG_PORT) {
					MAILBOX_t *slimmb;
					uint32_t slimword0;
					/* Check real SLIM for any errors */
					slimword0 = readl(phba->MBslimaddr);
					slimmb = (MAILBOX_t *) & slimword0;
					if (((slimword0 & OWN_CHIP) != OWN_CHIP)
					    && slimmb->mbxStatus) {
						psli->sli_flag &=
						    ~LPFC_SLI_ACTIVE;
						word0 = slimword0;
					}
				}
			} else {
				/* First copy command data */
				word0 = readl(phba->MBslimaddr);
			}
			/* Read the HBA Host Attention Register */
			if (lpfc_readl(phba->HAregaddr, &ha_copy)) {
				spin_unlock_irqrestore(&phba->hbalock,
						       drvr_flag);
				goto out_not_finished;
			}
		}

		if (psli->sli_flag & LPFC_SLI_ACTIVE) {
			/* copy results back to user */
			lpfc_sli_pcimem_bcopy(phba->mbox, mbx,
						MAILBOX_CMD_SIZE);
			/* Copy the mailbox extension data */
			if (pmbox->out_ext_byte_len && pmbox->ctx_buf) {
				lpfc_sli_pcimem_bcopy(phba->mbox_ext,
						      pmbox->ctx_buf,
						      pmbox->out_ext_byte_len);
			}
		} else {
			/* First copy command data */
			lpfc_memcpy_from_slim(mbx, phba->MBslimaddr,
						MAILBOX_CMD_SIZE);
			/* Copy the mailbox extension data */
			if (pmbox->out_ext_byte_len && pmbox->ctx_buf) {
				lpfc_memcpy_from_slim(
					pmbox->ctx_buf,
					phba->MBslimaddr +
					MAILBOX_HBA_EXT_OFFSET,
					pmbox->out_ext_byte_len);
			}
		}

		writel(HA_MBATT, phba->HAregaddr);
		readl(phba->HAregaddr); /* flush */

		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
		status = mbx->mbxStatus;
	}

	spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
	return status;

out_not_finished:
	if (processing_queue) {
		pmbox->u.mb.mbxStatus = MBX_NOT_FINISHED;
		lpfc_mbox_cmpl_put(phba, pmbox);
	}
	return MBX_NOT_FINISHED;
}

/**
 * lpfc_sli4_async_mbox_block - Block posting SLI4 asynchronous mailbox command
 * @phba: Pointer to HBA context object.
 *
 * The function blocks the posting of SLI4 asynchronous mailbox commands from
 * the driver internal pending mailbox queue. It will then try to wait out the
 * possible outstanding mailbox command before return.
 *
 * Returns:
 * 	0 - the outstanding mailbox command completed; otherwise, the wait for
 * 	the outstanding mailbox command timed out.
 **/
static int
lpfc_sli4_async_mbox_block(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;
	int rc = 0;
	unsigned long timeout = 0;

	/* Mark the asynchronous mailbox command posting as blocked */
	spin_lock_irq(&phba->hbalock);
	psli->sli_flag |= LPFC_SLI_ASYNC_MBX_BLK;
	/* Determine how long we might wait for the active mailbox
	 * command to be gracefully completed by firmware.
	 */
	if (phba->sli.mbox_active)
		timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba,
						phba->sli.mbox_active) *
						1000) + jiffies;
	spin_unlock_irq(&phba->hbalock);

	/* Make sure the mailbox is really active */
	if (timeout)
		lpfc_sli4_process_missed_mbox_completions(phba);

	/* Wait for the outstnading mailbox command to complete */
	while (phba->sli.mbox_active) {
		/* Check active mailbox complete status every 2ms */
		msleep(2);
		if (time_after(jiffies, timeout)) {
			/* Timeout, marked the outstanding cmd not complete */
			rc = 1;
			break;
		}
	}

	/* Can not cleanly block async mailbox command, fails it */
	if (rc) {
		spin_lock_irq(&phba->hbalock);
		psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
		spin_unlock_irq(&phba->hbalock);
	}
	return rc;
}

/**
 * lpfc_sli4_async_mbox_unblock - Block posting SLI4 async mailbox command
 * @phba: Pointer to HBA context object.
 *
 * The function unblocks and resume posting of SLI4 asynchronous mailbox
 * commands from the driver internal pending mailbox queue. It makes sure
 * that there is no outstanding mailbox command before resuming posting
 * asynchronous mailbox commands. If, for any reason, there is outstanding
 * mailbox command, it will try to wait it out before resuming asynchronous
 * mailbox command posting.
 **/
static void
lpfc_sli4_async_mbox_unblock(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;

	spin_lock_irq(&phba->hbalock);
	if (!(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
		/* Asynchronous mailbox posting is not blocked, do nothing */
		spin_unlock_irq(&phba->hbalock);
		return;
	}

	/* Outstanding synchronous mailbox command is guaranteed to be done,
	 * successful or timeout, after timing-out the outstanding mailbox
	 * command shall always be removed, so just unblock posting async
	 * mailbox command and resume
	 */
	psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
	spin_unlock_irq(&phba->hbalock);

	/* wake up worker thread to post asynchronlous mailbox command */
	lpfc_worker_wake_up(phba);
}

/**
 * lpfc_sli4_wait_bmbx_ready - Wait for bootstrap mailbox register ready
 * @phba: Pointer to HBA context object.
 * @mboxq: Pointer to mailbox object.
 *
 * The function waits for the bootstrap mailbox register ready bit from
 * port for twice the regular mailbox command timeout value.
 *
 *      0 - no timeout on waiting for bootstrap mailbox register ready.
 *      MBXERR_ERROR - wait for bootstrap mailbox register timed out.
 **/
static int
lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
{
	uint32_t db_ready;
	unsigned long timeout;
	struct lpfc_register bmbx_reg;

	timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, mboxq)
				   * 1000) + jiffies;

	do {
		bmbx_reg.word0 = readl(phba->sli4_hba.BMBXregaddr);
		db_ready = bf_get(lpfc_bmbx_rdy, &bmbx_reg);
		if (!db_ready)
			mdelay(2);

		if (time_after(jiffies, timeout))
			return MBXERR_ERROR;
	} while (!db_ready);

	return 0;
}

/**
 * lpfc_sli4_post_sync_mbox - Post an SLI4 mailbox to the bootstrap mailbox
 * @phba: Pointer to HBA context object.
 * @mboxq: Pointer to mailbox object.
 *
 * The function posts a mailbox to the port.  The mailbox is expected
 * to be comletely filled in and ready for the port to operate on it.
 * This routine executes a synchronous completion operation on the
 * mailbox by polling for its completion.
 *
 * The caller must not be holding any locks when calling this routine.
 *
 * Returns:
 *	MBX_SUCCESS - mailbox posted successfully
 *	Any of the MBX error values.
 **/
static int
lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
{
	int rc = MBX_SUCCESS;
	unsigned long iflag;
	uint32_t mcqe_status;
	uint32_t mbx_cmnd;
	struct lpfc_sli *psli = &phba->sli;
	struct lpfc_mqe *mb = &mboxq->u.mqe;
	struct lpfc_bmbx_create *mbox_rgn;
	struct dma_address *dma_address;

	/*
	 * Only one mailbox can be active to the bootstrap mailbox region
	 * at a time and there is no queueing provided.
	 */
	spin_lock_irqsave(&phba->hbalock, iflag);
	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
		spin_unlock_irqrestore(&phba->hbalock, iflag);
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"(%d):2532 Mailbox command x%x (x%x/x%x) "
				"cannot issue Data: x%x x%x\n",
				mboxq->vport ? mboxq->vport->vpi : 0,
				mboxq->u.mb.mbxCommand,
				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
				psli->sli_flag, MBX_POLL);
		return MBXERR_ERROR;
	}
	/* The server grabs the token and owns it until release */
	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;
	phba->sli.mbox_active = mboxq;
	spin_unlock_irqrestore(&phba->hbalock, iflag);

	/* wait for bootstrap mbox register for readyness */
	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
	if (rc)
		goto exit;
	/*
	 * Initialize the bootstrap memory region to avoid stale data areas
	 * in the mailbox post.  Then copy the caller's mailbox contents to
	 * the bmbx mailbox region.
	 */
	mbx_cmnd = bf_get(lpfc_mqe_command, mb);
	memset(phba->sli4_hba.bmbx.avirt, 0, sizeof(struct lpfc_bmbx_create));
	lpfc_sli4_pcimem_bcopy(mb, phba->sli4_hba.bmbx.avirt,
			       sizeof(struct lpfc_mqe));

	/* Post the high mailbox dma address to the port and wait for ready. */
	dma_address = &phba->sli4_hba.bmbx.dma_address;
	writel(dma_address->addr_hi, phba->sli4_hba.BMBXregaddr);

	/* wait for bootstrap mbox register for hi-address write done */
	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
	if (rc)
		goto exit;

	/* Post the low mailbox dma address to the port. */
	writel(dma_address->addr_lo, phba->sli4_hba.BMBXregaddr);

	/* wait for bootstrap mbox register for low address write done */
	rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq);
	if (rc)
		goto exit;

	/*
	 * Read the CQ to ensure the mailbox has completed.
	 * If so, update the mailbox status so that the upper layers
	 * can complete the request normally.
	 */
	lpfc_sli4_pcimem_bcopy(phba->sli4_hba.bmbx.avirt, mb,
			       sizeof(struct lpfc_mqe));
	mbox_rgn = (struct lpfc_bmbx_create *) phba->sli4_hba.bmbx.avirt;
	lpfc_sli4_pcimem_bcopy(&mbox_rgn->mcqe, &mboxq->mcqe,
			       sizeof(struct lpfc_mcqe));
	mcqe_status = bf_get(lpfc_mcqe_status, &mbox_rgn->mcqe);
	/*
	 * When the CQE status indicates a failure and the mailbox status
	 * indicates success then copy the CQE status into the mailbox status
	 * (and prefix it with x4000).
	 */
	if (mcqe_status != MB_CQE_STATUS_SUCCESS) {
		if (bf_get(lpfc_mqe_status, mb) == MBX_SUCCESS)
			bf_set(lpfc_mqe_status, mb,
			       (LPFC_MBX_ERROR_RANGE | mcqe_status));
		rc = MBXERR_ERROR;
	} else
		lpfc_sli4_swap_str(phba, mboxq);

	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
			"(%d):0356 Mailbox cmd x%x (x%x/x%x) Status x%x "
			"Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x x%x x%x"
			" x%x x%x CQ: x%x x%x x%x x%x\n",
			mboxq->vport ? mboxq->vport->vpi : 0, mbx_cmnd,
			lpfc_sli_config_mbox_subsys_get(phba, mboxq),
			lpfc_sli_config_mbox_opcode_get(phba, mboxq),
			bf_get(lpfc_mqe_status, mb),
			mb->un.mb_words[0], mb->un.mb_words[1],
			mb->un.mb_words[2], mb->un.mb_words[3],
			mb->un.mb_words[4], mb->un.mb_words[5],
			mb->un.mb_words[6], mb->un.mb_words[7],
			mb->un.mb_words[8], mb->un.mb_words[9],
			mb->un.mb_words[10], mb->un.mb_words[11],
			mb->un.mb_words[12], mboxq->mcqe.word0,
			mboxq->mcqe.mcqe_tag0, 	mboxq->mcqe.mcqe_tag1,
			mboxq->mcqe.trailer);
exit:
	/* We are holding the token, no needed for lock when release */
	spin_lock_irqsave(&phba->hbalock, iflag);
	psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
	phba->sli.mbox_active = NULL;
	spin_unlock_irqrestore(&phba->hbalock, iflag);
	return rc;
}

/**
 * lpfc_sli_issue_mbox_s4 - Issue an SLI4 mailbox command to firmware
 * @phba: Pointer to HBA context object.
 * @pmbox: Pointer to mailbox object.
 * @flag: Flag indicating how the mailbox need to be processed.
 *
 * This function is called by discovery code and HBA management code to submit
 * a mailbox command to firmware with SLI-4 interface spec.
 *
 * Return codes the caller owns the mailbox command after the return of the
 * function.
 **/
static int
lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
		       uint32_t flag)
{
	struct lpfc_sli *psli = &phba->sli;
	unsigned long iflags;
	int rc;

	/* dump from issue mailbox command if setup */
	lpfc_idiag_mbxacc_dump_issue_mbox(phba, &mboxq->u.mb);

	rc = lpfc_mbox_dev_check(phba);
	if (unlikely(rc)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"(%d):2544 Mailbox command x%x (x%x/x%x) "
				"cannot issue Data: x%x x%x\n",
				mboxq->vport ? mboxq->vport->vpi : 0,
				mboxq->u.mb.mbxCommand,
				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
				psli->sli_flag, flag);
		goto out_not_finished;
	}

	/* Detect polling mode and jump to a handler */
	if (!phba->sli4_hba.intr_enable) {
		if (flag == MBX_POLL)
			rc = lpfc_sli4_post_sync_mbox(phba, mboxq);
		else
			rc = -EIO;
		if (rc != MBX_SUCCESS)
			lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
					"(%d):2541 Mailbox command x%x "
					"(x%x/x%x) failure: "
					"mqe_sta: x%x mcqe_sta: x%x/x%x "
					"Data: x%x x%x\n,",
					mboxq->vport ? mboxq->vport->vpi : 0,
					mboxq->u.mb.mbxCommand,
					lpfc_sli_config_mbox_subsys_get(phba,
									mboxq),
					lpfc_sli_config_mbox_opcode_get(phba,
									mboxq),
					bf_get(lpfc_mqe_status, &mboxq->u.mqe),
					bf_get(lpfc_mcqe_status, &mboxq->mcqe),
					bf_get(lpfc_mcqe_ext_status,
					       &mboxq->mcqe),
					psli->sli_flag, flag);
		return rc;
	} else if (flag == MBX_POLL) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
				"(%d):2542 Try to issue mailbox command "
				"x%x (x%x/x%x) synchronously ahead of async "
				"mailbox command queue: x%x x%x\n",
				mboxq->vport ? mboxq->vport->vpi : 0,
				mboxq->u.mb.mbxCommand,
				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
				psli->sli_flag, flag);
		/* Try to block the asynchronous mailbox posting */
		rc = lpfc_sli4_async_mbox_block(phba);
		if (!rc) {
			/* Successfully blocked, now issue sync mbox cmd */
			rc = lpfc_sli4_post_sync_mbox(phba, mboxq);
			if (rc != MBX_SUCCESS)
				lpfc_printf_log(phba, KERN_WARNING,
					LOG_MBOX | LOG_SLI,
					"(%d):2597 Sync Mailbox command "
					"x%x (x%x/x%x) failure: "
					"mqe_sta: x%x mcqe_sta: x%x/x%x "
					"Data: x%x x%x\n,",
					mboxq->vport ? mboxq->vport->vpi : 0,
					mboxq->u.mb.mbxCommand,
					lpfc_sli_config_mbox_subsys_get(phba,
									mboxq),
					lpfc_sli_config_mbox_opcode_get(phba,
									mboxq),
					bf_get(lpfc_mqe_status, &mboxq->u.mqe),
					bf_get(lpfc_mcqe_status, &mboxq->mcqe),
					bf_get(lpfc_mcqe_ext_status,
					       &mboxq->mcqe),
					psli->sli_flag, flag);
			/* Unblock the async mailbox posting afterward */
			lpfc_sli4_async_mbox_unblock(phba);
		}
		return rc;
	}

	/* Now, interrupt mode asynchrous mailbox command */
	rc = lpfc_mbox_cmd_check(phba, mboxq);
	if (rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"(%d):2543 Mailbox command x%x (x%x/x%x) "
				"cannot issue Data: x%x x%x\n",
				mboxq->vport ? mboxq->vport->vpi : 0,
				mboxq->u.mb.mbxCommand,
				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
				psli->sli_flag, flag);
		goto out_not_finished;
	}

	/* Put the mailbox command to the driver internal FIFO */
	psli->slistat.mbox_busy++;
	spin_lock_irqsave(&phba->hbalock, iflags);
	lpfc_mbox_put(phba, mboxq);
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
			"(%d):0354 Mbox cmd issue - Enqueue Data: "
			"x%x (x%x/x%x) x%x x%x x%x\n",
			mboxq->vport ? mboxq->vport->vpi : 0xffffff,
			bf_get(lpfc_mqe_command, &mboxq->u.mqe),
			lpfc_sli_config_mbox_subsys_get(phba, mboxq),
			lpfc_sli_config_mbox_opcode_get(phba, mboxq),
			phba->pport->port_state,
			psli->sli_flag, MBX_NOWAIT);
	/* Wake up worker thread to transport mailbox command from head */
	lpfc_worker_wake_up(phba);

	return MBX_BUSY;

out_not_finished:
	return MBX_NOT_FINISHED;
}

/**
 * lpfc_sli4_post_async_mbox - Post an SLI4 mailbox command to device
 * @phba: Pointer to HBA context object.
 *
 * This function is called by worker thread to send a mailbox command to
 * SLI4 HBA firmware.
 *
 **/
int
lpfc_sli4_post_async_mbox(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;
	LPFC_MBOXQ_t *mboxq;
	int rc = MBX_SUCCESS;
	unsigned long iflags;
	struct lpfc_mqe *mqe;
	uint32_t mbx_cmnd;

	/* Check interrupt mode before post async mailbox command */
	if (unlikely(!phba->sli4_hba.intr_enable))
		return MBX_NOT_FINISHED;

	/* Check for mailbox command service token */
	spin_lock_irqsave(&phba->hbalock, iflags);
	if (unlikely(psli->sli_flag & LPFC_SLI_ASYNC_MBX_BLK)) {
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		return MBX_NOT_FINISHED;
	}
	if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) {
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		return MBX_NOT_FINISHED;
	}
	if (unlikely(phba->sli.mbox_active)) {
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"0384 There is pending active mailbox cmd\n");
		return MBX_NOT_FINISHED;
	}
	/* Take the mailbox command service token */
	psli->sli_flag |= LPFC_SLI_MBOX_ACTIVE;

	/* Get the next mailbox command from head of queue */
	mboxq = lpfc_mbox_get(phba);

	/* If no more mailbox command waiting for post, we're done */
	if (!mboxq) {
		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		return MBX_SUCCESS;
	}
	phba->sli.mbox_active = mboxq;
	spin_unlock_irqrestore(&phba->hbalock, iflags);

	/* Check device readiness for posting mailbox command */
	rc = lpfc_mbox_dev_check(phba);
	if (unlikely(rc))
		/* Driver clean routine will clean up pending mailbox */
		goto out_not_finished;

	/* Prepare the mbox command to be posted */
	mqe = &mboxq->u.mqe;
	mbx_cmnd = bf_get(lpfc_mqe_command, mqe);

	/* Start timer for the mbox_tmo and log some mailbox post messages */
	mod_timer(&psli->mbox_tmo, (jiffies +
		  msecs_to_jiffies(1000 * lpfc_mbox_tmo_val(phba, mboxq))));

	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
			"(%d):0355 Mailbox cmd x%x (x%x/x%x) issue Data: "
			"x%x x%x\n",
			mboxq->vport ? mboxq->vport->vpi : 0, mbx_cmnd,
			lpfc_sli_config_mbox_subsys_get(phba, mboxq),
			lpfc_sli_config_mbox_opcode_get(phba, mboxq),
			phba->pport->port_state, psli->sli_flag);

	if (mbx_cmnd != MBX_HEARTBEAT) {
		if (mboxq->vport) {
			lpfc_debugfs_disc_trc(mboxq->vport,
				LPFC_DISC_TRC_MBOX_VPORT,
				"MBOX Send vport: cmd:x%x mb:x%x x%x",
				mbx_cmnd, mqe->un.mb_words[0],
				mqe->un.mb_words[1]);
		} else {
			lpfc_debugfs_disc_trc(phba->pport,
				LPFC_DISC_TRC_MBOX,
				"MBOX Send: cmd:x%x mb:x%x x%x",
				mbx_cmnd, mqe->un.mb_words[0],
				mqe->un.mb_words[1]);
		}
	}
	psli->slistat.mbox_cmd++;

	/* Post the mailbox command to the port */
	rc = lpfc_sli4_mq_put(phba->sli4_hba.mbx_wq, mqe);
	if (rc != MBX_SUCCESS) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
				"(%d):2533 Mailbox command x%x (x%x/x%x) "
				"cannot issue Data: x%x x%x\n",
				mboxq->vport ? mboxq->vport->vpi : 0,
				mboxq->u.mb.mbxCommand,
				lpfc_sli_config_mbox_subsys_get(phba, mboxq),
				lpfc_sli_config_mbox_opcode_get(phba, mboxq),
				psli->sli_flag, MBX_NOWAIT);
		goto out_not_finished;
	}

	return rc;

out_not_finished:
	spin_lock_irqsave(&phba->hbalock, iflags);
	if (phba->sli.mbox_active) {
		mboxq->u.mb.mbxStatus = MBX_NOT_FINISHED;
		__lpfc_mbox_cmpl_put(phba, mboxq);
		/* Release the token */
		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
		phba->sli.mbox_active = NULL;
	}
	spin_unlock_irqrestore(&phba->hbalock, iflags);

	return MBX_NOT_FINISHED;
}

/**
 * lpfc_sli_issue_mbox - Wrapper func for issuing mailbox command
 * @phba: Pointer to HBA context object.
 * @pmbox: Pointer to mailbox object.
 * @flag: Flag indicating how the mailbox need to be processed.
 *
 * This routine wraps the actual SLI3 or SLI4 mailbox issuing routine from
 * the API jump table function pointer from the lpfc_hba struct.
 *
 * Return codes the caller owns the mailbox command after the return of the
 * function.
 **/
int
lpfc_sli_issue_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, uint32_t flag)
{
	return phba->lpfc_sli_issue_mbox(phba, pmbox, flag);
}

/**
 * lpfc_mbox_api_table_setup - Set up mbox api function jump table
 * @phba: The hba struct for which this call is being executed.
 * @dev_grp: The HBA PCI-Device group number.
 *
 * This routine sets up the mbox interface API function jump table in @phba
 * struct.
 * Returns: 0 - success, -ENODEV - failure.
 **/
int
lpfc_mbox_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
{

	switch (dev_grp) {
	case LPFC_PCI_DEV_LP:
		phba->lpfc_sli_issue_mbox = lpfc_sli_issue_mbox_s3;
		phba->lpfc_sli_handle_slow_ring_event =
				lpfc_sli_handle_slow_ring_event_s3;
		phba->lpfc_sli_hbq_to_firmware = lpfc_sli_hbq_to_firmware_s3;
		phba->lpfc_sli_brdrestart = lpfc_sli_brdrestart_s3;
		phba->lpfc_sli_brdready = lpfc_sli_brdready_s3;
		break;
	case LPFC_PCI_DEV_OC:
		phba->lpfc_sli_issue_mbox = lpfc_sli_issue_mbox_s4;
		phba->lpfc_sli_handle_slow_ring_event =
				lpfc_sli_handle_slow_ring_event_s4;
		phba->lpfc_sli_hbq_to_firmware = lpfc_sli_hbq_to_firmware_s4;
		phba->lpfc_sli_brdrestart = lpfc_sli_brdrestart_s4;
		phba->lpfc_sli_brdready = lpfc_sli_brdready_s4;
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"1420 Invalid HBA PCI-device group: 0x%x\n",
				dev_grp);
		return -ENODEV;
		break;
	}
	return 0;
}

/**
 * __lpfc_sli_ringtx_put - Add an iocb to the txq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @piocb: Pointer to address of newly added command iocb.
 *
 * This function is called with hbalock held to add a command
 * iocb to the txq when SLI layer cannot submit the command iocb
 * to the ring.
 **/
void
__lpfc_sli_ringtx_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
		    struct lpfc_iocbq *piocb)
{
	lockdep_assert_held(&phba->hbalock);
	/* Insert the caller's iocb in the txq tail for later processing. */
	list_add_tail(&piocb->list, &pring->txq);
}

/**
 * lpfc_sli_next_iocb - Get the next iocb in the txq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @piocb: Pointer to address of newly added command iocb.
 *
 * This function is called with hbalock held before a new
 * iocb is submitted to the firmware. This function checks
 * txq to flush the iocbs in txq to Firmware before
 * submitting new iocbs to the Firmware.
 * If there are iocbs in the txq which need to be submitted
 * to firmware, lpfc_sli_next_iocb returns the first element
 * of the txq after dequeuing it from txq.
 * If there is no iocb in the txq then the function will return
 * *piocb and *piocb is set to NULL. Caller needs to check
 * *piocb to find if there are more commands in the txq.
 **/
static struct lpfc_iocbq *
lpfc_sli_next_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
		   struct lpfc_iocbq **piocb)
{
	struct lpfc_iocbq * nextiocb;

	lockdep_assert_held(&phba->hbalock);

	nextiocb = lpfc_sli_ringtx_get(phba, pring);
	if (!nextiocb) {
		nextiocb = *piocb;
		*piocb = NULL;
	}

	return nextiocb;
}

/**
 * __lpfc_sli_issue_iocb_s3 - SLI3 device lockless ver of lpfc_sli_issue_iocb
 * @phba: Pointer to HBA context object.
 * @ring_number: SLI ring number to issue iocb on.
 * @piocb: Pointer to command iocb.
 * @flag: Flag indicating if this command can be put into txq.
 *
 * __lpfc_sli_issue_iocb_s3 is used by other functions in the driver to issue
 * an iocb command to an HBA with SLI-3 interface spec. If the PCI slot is
 * recovering from error state, if HBA is resetting or if LPFC_STOP_IOCB_EVENT
 * flag is turned on, the function returns IOCB_ERROR. When the link is down,
 * this function allows only iocbs for posting buffers. This function finds
 * next available slot in the command ring and posts the command to the
 * available slot and writes the port attention register to request HBA start
 * processing new iocb. If there is no slot available in the ring and
 * flag & SLI_IOCB_RET_IOCB is set, the new iocb is added to the txq, otherwise
 * the function returns IOCB_BUSY.
 *
 * This function is called with hbalock held. The function will return success
 * after it successfully submit the iocb to firmware or after adding to the
 * txq.
 **/
static int
__lpfc_sli_issue_iocb_s3(struct lpfc_hba *phba, uint32_t ring_number,
		    struct lpfc_iocbq *piocb, uint32_t flag)
{
	struct lpfc_iocbq *nextiocb;
	IOCB_t *iocb;
	struct lpfc_sli_ring *pring = &phba->sli.sli3_ring[ring_number];

	lockdep_assert_held(&phba->hbalock);

	if (piocb->iocb_cmpl && (!piocb->vport) &&
	   (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) &&
	   (piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) {
		lpfc_printf_log(phba, KERN_ERR,
				LOG_SLI | LOG_VPORT,
				"1807 IOCB x%x failed. No vport\n",
				piocb->iocb.ulpCommand);
		dump_stack();
		return IOCB_ERROR;
	}


	/* If the PCI channel is in offline state, do not post iocbs. */
	if (unlikely(pci_channel_offline(phba->pcidev)))
		return IOCB_ERROR;

	/* If HBA has a deferred error attention, fail the iocb. */
	if (unlikely(phba->hba_flag & DEFER_ERATT))
		return IOCB_ERROR;

	/*
	 * We should never get an IOCB if we are in a < LINK_DOWN state
	 */
	if (unlikely(phba->link_state < LPFC_LINK_DOWN))
		return IOCB_ERROR;

	/*
	 * Check to see if we are blocking IOCB processing because of a
	 * outstanding event.
	 */
	if (unlikely(pring->flag & LPFC_STOP_IOCB_EVENT))
		goto iocb_busy;

	if (unlikely(phba->link_state == LPFC_LINK_DOWN)) {
		/*
		 * Only CREATE_XRI, CLOSE_XRI, and QUE_RING_BUF
		 * can be issued if the link is not up.
		 */
		switch (piocb->iocb.ulpCommand) {
		case CMD_GEN_REQUEST64_CR:
		case CMD_GEN_REQUEST64_CX:
			if (!(phba->sli.sli_flag & LPFC_MENLO_MAINT) ||
				(piocb->iocb.un.genreq64.w5.hcsw.Rctl !=
					FC_RCTL_DD_UNSOL_CMD) ||
				(piocb->iocb.un.genreq64.w5.hcsw.Type !=
					MENLO_TRANSPORT_TYPE))

				goto iocb_busy;
			break;
		case CMD_QUE_RING_BUF_CN:
		case CMD_QUE_RING_BUF64_CN:
			/*
			 * For IOCBs, like QUE_RING_BUF, that have no rsp ring
			 * completion, iocb_cmpl MUST be 0.
			 */
			if (piocb->iocb_cmpl)
				piocb->iocb_cmpl = NULL;
			/*FALLTHROUGH*/
		case CMD_CREATE_XRI_CR:
		case CMD_CLOSE_XRI_CN:
		case CMD_CLOSE_XRI_CX:
			break;
		default:
			goto iocb_busy;
		}

	/*
	 * For FCP commands, we must be in a state where we can process link
	 * attention events.
	 */
	} else if (unlikely(pring->ringno == LPFC_FCP_RING &&
			    !(phba->sli.sli_flag & LPFC_PROCESS_LA))) {
		goto iocb_busy;
	}

	while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
	       (nextiocb = lpfc_sli_next_iocb(phba, pring, &piocb)))
		lpfc_sli_submit_iocb(phba, pring, iocb, nextiocb);

	if (iocb)
		lpfc_sli_update_ring(phba, pring);
	else
		lpfc_sli_update_full_ring(phba, pring);

	if (!piocb)
		return IOCB_SUCCESS;

	goto out_busy;

 iocb_busy:
	pring->stats.iocb_cmd_delay++;

 out_busy:

	if (!(flag & SLI_IOCB_RET_IOCB)) {
		__lpfc_sli_ringtx_put(phba, pring, piocb);
		return IOCB_SUCCESS;
	}

	return IOCB_BUSY;
}

/**
 * lpfc_sli4_bpl2sgl - Convert the bpl/bde to a sgl.
 * @phba: Pointer to HBA context object.
 * @piocb: Pointer to command iocb.
 * @sglq: Pointer to the scatter gather queue object.
 *
 * This routine converts the bpl or bde that is in the IOCB
 * to a sgl list for the sli4 hardware. The physical address
 * of the bpl/bde is converted back to a virtual address.
 * If the IOCB contains a BPL then the list of BDE's is
 * converted to sli4_sge's. If the IOCB contains a single
 * BDE then it is converted to a single sli_sge.
 * The IOCB is still in cpu endianess so the contents of
 * the bpl can be used without byte swapping.
 *
 * Returns valid XRI = Success, NO_XRI = Failure.
**/
static uint16_t
lpfc_sli4_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq,
		struct lpfc_sglq *sglq)
{
	uint16_t xritag = NO_XRI;
	struct ulp_bde64 *bpl = NULL;
	struct ulp_bde64 bde;
	struct sli4_sge *sgl  = NULL;
	struct lpfc_dmabuf *dmabuf;
	IOCB_t *icmd;
	int numBdes = 0;
	int i = 0;
	uint32_t offset = 0; /* accumulated offset in the sg request list */
	int inbound = 0; /* number of sg reply entries inbound from firmware */

	if (!piocbq || !sglq)
		return xritag;

	sgl  = (struct sli4_sge *)sglq->sgl;
	icmd = &piocbq->iocb;
	if (icmd->ulpCommand == CMD_XMIT_BLS_RSP64_CX)
		return sglq->sli4_xritag;
	if (icmd->un.genreq64.bdl.bdeFlags == BUFF_TYPE_BLP_64) {
		numBdes = icmd->un.genreq64.bdl.bdeSize /
				sizeof(struct ulp_bde64);
		/* The addrHigh and addrLow fields within the IOCB
		 * have not been byteswapped yet so there is no
		 * need to swap them back.
		 */
		if (piocbq->context3)
			dmabuf = (struct lpfc_dmabuf *)piocbq->context3;
		else
			return xritag;

		bpl  = (struct ulp_bde64 *)dmabuf->virt;
		if (!bpl)
			return xritag;

		for (i = 0; i < numBdes; i++) {
			/* Should already be byte swapped. */
			sgl->addr_hi = bpl->addrHigh;
			sgl->addr_lo = bpl->addrLow;

			sgl->word2 = le32_to_cpu(sgl->word2);
			if ((i+1) == numBdes)
				bf_set(lpfc_sli4_sge_last, sgl, 1);
			else
				bf_set(lpfc_sli4_sge_last, sgl, 0);
			/* swap the size field back to the cpu so we
			 * can assign it to the sgl.
			 */
			bde.tus.w = le32_to_cpu(bpl->tus.w);
			sgl->sge_len = cpu_to_le32(bde.tus.f.bdeSize);
			/* The offsets in the sgl need to be accumulated
			 * separately for the request and reply lists.
			 * The request is always first, the reply follows.
			 */
			if (piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) {
				/* add up the reply sg entries */
				if (bpl->tus.f.bdeFlags == BUFF_TYPE_BDE_64I)
					inbound++;
				/* first inbound? reset the offset */
				if (inbound == 1)
					offset = 0;
				bf_set(lpfc_sli4_sge_offset, sgl, offset);
				bf_set(lpfc_sli4_sge_type, sgl,
					LPFC_SGE_TYPE_DATA);
				offset += bde.tus.f.bdeSize;
			}
			sgl->word2 = cpu_to_le32(sgl->word2);
			bpl++;
			sgl++;
		}
	} else if (icmd->un.genreq64.bdl.bdeFlags == BUFF_TYPE_BDE_64) {
			/* The addrHigh and addrLow fields of the BDE have not
			 * been byteswapped yet so they need to be swapped
			 * before putting them in the sgl.
			 */
			sgl->addr_hi =
				cpu_to_le32(icmd->un.genreq64.bdl.addrHigh);
			sgl->addr_lo =
				cpu_to_le32(icmd->un.genreq64.bdl.addrLow);
			sgl->word2 = le32_to_cpu(sgl->word2);
			bf_set(lpfc_sli4_sge_last, sgl, 1);
			sgl->word2 = cpu_to_le32(sgl->word2);
			sgl->sge_len =
				cpu_to_le32(icmd->un.genreq64.bdl.bdeSize);
	}
	return sglq->sli4_xritag;
}

/**
 * lpfc_sli_iocb2wqe - Convert the IOCB to a work queue entry.
 * @phba: Pointer to HBA context object.
 * @piocb: Pointer to command iocb.
 * @wqe: Pointer to the work queue entry.
 *
 * This routine converts the iocb command to its Work Queue Entry
 * equivalent. The wqe pointer should not have any fields set when
 * this routine is called because it will memcpy over them.
 * This routine does not set the CQ_ID or the WQEC bits in the
 * wqe.
 *
 * Returns: 0 = Success, IOCB_ERROR = Failure.
 **/
static int
lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
		union lpfc_wqe128 *wqe)
{
	uint32_t xmit_len = 0, total_len = 0;
	uint8_t ct = 0;
	uint32_t fip;
	uint32_t abort_tag;
	uint8_t command_type = ELS_COMMAND_NON_FIP;
	uint8_t cmnd;
	uint16_t xritag;
	uint16_t abrt_iotag;
	struct lpfc_iocbq *abrtiocbq;
	struct ulp_bde64 *bpl = NULL;
	uint32_t els_id = LPFC_ELS_ID_DEFAULT;
	int numBdes, i;
	struct ulp_bde64 bde;
	struct lpfc_nodelist *ndlp;
	uint32_t *pcmd;
	uint32_t if_type;

	fip = phba->hba_flag & HBA_FIP_SUPPORT;
	/* The fcp commands will set command type */
	if (iocbq->iocb_flag &  LPFC_IO_FCP)
		command_type = FCP_COMMAND;
	else if (fip && (iocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK))
		command_type = ELS_COMMAND_FIP;
	else
		command_type = ELS_COMMAND_NON_FIP;

	if (phba->fcp_embed_io)
		memset(wqe, 0, sizeof(union lpfc_wqe128));
	/* Some of the fields are in the right position already */
	memcpy(wqe, &iocbq->iocb, sizeof(union lpfc_wqe));
	if (iocbq->iocb.ulpCommand != CMD_SEND_FRAME) {
		/* The ct field has moved so reset */
		wqe->generic.wqe_com.word7 = 0;
		wqe->generic.wqe_com.word10 = 0;
	}

	abort_tag = (uint32_t) iocbq->iotag;
	xritag = iocbq->sli4_xritag;
	/* words0-2 bpl convert bde */
	if (iocbq->iocb.un.genreq64.bdl.bdeFlags == BUFF_TYPE_BLP_64) {
		numBdes = iocbq->iocb.un.genreq64.bdl.bdeSize /
				sizeof(struct ulp_bde64);
		bpl  = (struct ulp_bde64 *)
			((struct lpfc_dmabuf *)iocbq->context3)->virt;
		if (!bpl)
			return IOCB_ERROR;

		/* Should already be byte swapped. */
		wqe->generic.bde.addrHigh =  le32_to_cpu(bpl->addrHigh);
		wqe->generic.bde.addrLow =  le32_to_cpu(bpl->addrLow);
		/* swap the size field back to the cpu so we
		 * can assign it to the sgl.
		 */
		wqe->generic.bde.tus.w  = le32_to_cpu(bpl->tus.w);
		xmit_len = wqe->generic.bde.tus.f.bdeSize;
		total_len = 0;
		for (i = 0; i < numBdes; i++) {
			bde.tus.w  = le32_to_cpu(bpl[i].tus.w);
			total_len += bde.tus.f.bdeSize;
		}
	} else
		xmit_len = iocbq->iocb.un.fcpi64.bdl.bdeSize;

	iocbq->iocb.ulpIoTag = iocbq->iotag;
	cmnd = iocbq->iocb.ulpCommand;

	switch (iocbq->iocb.ulpCommand) {
	case CMD_ELS_REQUEST64_CR:
		if (iocbq->iocb_flag & LPFC_IO_LIBDFC)
			ndlp = iocbq->context_un.ndlp;
		else
			ndlp = (struct lpfc_nodelist *)iocbq->context1;
		if (!iocbq->iocb.ulpLe) {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"2007 Only Limited Edition cmd Format"
				" supported 0x%x\n",
				iocbq->iocb.ulpCommand);
			return IOCB_ERROR;
		}

		wqe->els_req.payload_len = xmit_len;
		/* Els_reguest64 has a TMO */
		bf_set(wqe_tmo, &wqe->els_req.wqe_com,
			iocbq->iocb.ulpTimeout);
		/* Need a VF for word 4 set the vf bit*/
		bf_set(els_req64_vf, &wqe->els_req, 0);
		/* And a VFID for word 12 */
		bf_set(els_req64_vfid, &wqe->els_req, 0);
		ct = ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l);
		bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
		       iocbq->iocb.ulpContext);
		bf_set(wqe_ct, &wqe->els_req.wqe_com, ct);
		bf_set(wqe_pu, &wqe->els_req.wqe_com, 0);
		/* CCP CCPE PV PRI in word10 were set in the memcpy */
		if (command_type == ELS_COMMAND_FIP)
			els_id = ((iocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK)
					>> LPFC_FIP_ELS_ID_SHIFT);
		pcmd = (uint32_t *) (((struct lpfc_dmabuf *)
					iocbq->context2)->virt);
		if_type = bf_get(lpfc_sli_intf_if_type,
					&phba->sli4_hba.sli_intf);
		if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
			if (pcmd && (*pcmd == ELS_CMD_FLOGI ||
				*pcmd == ELS_CMD_SCR ||
				*pcmd == ELS_CMD_FDISC ||
				*pcmd == ELS_CMD_LOGO ||
				*pcmd == ELS_CMD_PLOGI)) {
				bf_set(els_req64_sp, &wqe->els_req, 1);
				bf_set(els_req64_sid, &wqe->els_req,
					iocbq->vport->fc_myDID);
				if ((*pcmd == ELS_CMD_FLOGI) &&
					!(phba->fc_topology ==
						LPFC_TOPOLOGY_LOOP))
					bf_set(els_req64_sid, &wqe->els_req, 0);
				bf_set(wqe_ct, &wqe->els_req.wqe_com, 1);
				bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
					phba->vpi_ids[iocbq->vport->vpi]);
			} else if (pcmd && iocbq->context1) {
				bf_set(wqe_ct, &wqe->els_req.wqe_com, 0);
				bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com,
					phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
			}
		}
		bf_set(wqe_temp_rpi, &wqe->els_req.wqe_com,
		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
		bf_set(wqe_els_id, &wqe->els_req.wqe_com, els_id);
		bf_set(wqe_dbde, &wqe->els_req.wqe_com, 1);
		bf_set(wqe_iod, &wqe->els_req.wqe_com, LPFC_WQE_IOD_READ);
		bf_set(wqe_qosd, &wqe->els_req.wqe_com, 1);
		bf_set(wqe_lenloc, &wqe->els_req.wqe_com, LPFC_WQE_LENLOC_NONE);
		bf_set(wqe_ebde_cnt, &wqe->els_req.wqe_com, 0);
		wqe->els_req.max_response_payload_len = total_len - xmit_len;
		break;
	case CMD_XMIT_SEQUENCE64_CX:
		bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
		       iocbq->iocb.un.ulpWord[3]);
		bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com,
		       iocbq->iocb.unsli3.rcvsli3.ox_id);
		/* The entire sequence is transmitted for this IOCB */
		xmit_len = total_len;
		cmnd = CMD_XMIT_SEQUENCE64_CR;
		if (phba->link_flag & LS_LOOPBACK_MODE)
			bf_set(wqe_xo, &wqe->xmit_sequence.wge_ctl, 1);
		/* fall through */
	case CMD_XMIT_SEQUENCE64_CR:
		/* word3 iocb=io_tag32 wqe=reserved */
		wqe->xmit_sequence.rsvd3 = 0;
		/* word4 relative_offset memcpy */
		/* word5 r_ctl/df_ctl memcpy */
		bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
		bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
		bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com,
		       LPFC_WQE_IOD_WRITE);
		bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
		       LPFC_WQE_LENLOC_WORD12);
		bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
		wqe->xmit_sequence.xmit_len = xmit_len;
		command_type = OTHER_COMMAND;
		break;
	case CMD_XMIT_BCAST64_CN:
		/* word3 iocb=iotag32 wqe=seq_payload_len */
		wqe->xmit_bcast64.seq_payload_len = xmit_len;
		/* word4 iocb=rsvd wqe=rsvd */
		/* word5 iocb=rctl/type/df_ctl wqe=rctl/type/df_ctl memcpy */
		/* word6 iocb=ctxt_tag/io_tag wqe=ctxt_tag/xri */
		bf_set(wqe_ct, &wqe->xmit_bcast64.wqe_com,
			((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l));
		bf_set(wqe_dbde, &wqe->xmit_bcast64.wqe_com, 1);
		bf_set(wqe_iod, &wqe->xmit_bcast64.wqe_com, LPFC_WQE_IOD_WRITE);
		bf_set(wqe_lenloc, &wqe->xmit_bcast64.wqe_com,
		       LPFC_WQE_LENLOC_WORD3);
		bf_set(wqe_ebde_cnt, &wqe->xmit_bcast64.wqe_com, 0);
		break;
	case CMD_FCP_IWRITE64_CR:
		command_type = FCP_COMMAND_DATA_OUT;
		/* word3 iocb=iotag wqe=payload_offset_len */
		/* Add the FCP_CMD and FCP_RSP sizes to get the offset */
		bf_set(payload_offset_len, &wqe->fcp_iwrite,
		       xmit_len + sizeof(struct fcp_rsp));
		bf_set(cmd_buff_len, &wqe->fcp_iwrite,
		       0);
		/* word4 iocb=parameter wqe=total_xfer_length memcpy */
		/* word5 iocb=initial_xfer_len wqe=initial_xfer_len memcpy */
		bf_set(wqe_erp, &wqe->fcp_iwrite.wqe_com,
		       iocbq->iocb.ulpFCP2Rcvy);
		bf_set(wqe_lnk, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpXS);
		/* Always open the exchange */
		bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
		bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com,
		       LPFC_WQE_LENLOC_WORD4);
		bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpPU);
		bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 1);
		if (iocbq->iocb_flag & LPFC_IO_OAS) {
			bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1);
			bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
			if (iocbq->priority) {
				bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
				       (iocbq->priority << 1));
			} else {
				bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
				       (phba->cfg_XLanePriority << 1));
			}
		}
		/* Note, word 10 is already initialized to 0 */

		/* Don't set PBDE for Perf hints, just lpfc_enable_pbde */
		if (phba->cfg_enable_pbde)
			bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
		else
			bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0);

		if (phba->fcp_embed_io) {
			struct lpfc_io_buf *lpfc_cmd;
			struct sli4_sge *sgl;
			struct fcp_cmnd *fcp_cmnd;
			uint32_t *ptr;

			/* 128 byte wqe support here */

			lpfc_cmd = iocbq->context1;
			sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
			fcp_cmnd = lpfc_cmd->fcp_cmnd;

			/* Word 0-2 - FCP_CMND */
			wqe->generic.bde.tus.f.bdeFlags =
				BUFF_TYPE_BDE_IMMED;
			wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
			wqe->generic.bde.addrHigh = 0;
			wqe->generic.bde.addrLow =  88;  /* Word 22 */

			bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
			bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);

			/* Word 22-29  FCP CMND Payload */
			ptr = &wqe->words[22];
			memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
		}
		break;
	case CMD_FCP_IREAD64_CR:
		/* word3 iocb=iotag wqe=payload_offset_len */
		/* Add the FCP_CMD and FCP_RSP sizes to get the offset */
		bf_set(payload_offset_len, &wqe->fcp_iread,
		       xmit_len + sizeof(struct fcp_rsp));
		bf_set(cmd_buff_len, &wqe->fcp_iread,
		       0);
		/* word4 iocb=parameter wqe=total_xfer_length memcpy */
		/* word5 iocb=initial_xfer_len wqe=initial_xfer_len memcpy */
		bf_set(wqe_erp, &wqe->fcp_iread.wqe_com,
		       iocbq->iocb.ulpFCP2Rcvy);
		bf_set(wqe_lnk, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpXS);
		/* Always open the exchange */
		bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
		bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com,
		       LPFC_WQE_LENLOC_WORD4);
		bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpPU);
		bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 1);
		if (iocbq->iocb_flag & LPFC_IO_OAS) {
			bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1);
			bf_set(wqe_ccpe, &wqe->fcp_iread.wqe_com, 1);
			if (iocbq->priority) {
				bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com,
				       (iocbq->priority << 1));
			} else {
				bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com,
				       (phba->cfg_XLanePriority << 1));
			}
		}
		/* Note, word 10 is already initialized to 0 */

		/* Don't set PBDE for Perf hints, just lpfc_enable_pbde */
		if (phba->cfg_enable_pbde)
			bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
		else
			bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0);

		if (phba->fcp_embed_io) {
			struct lpfc_io_buf *lpfc_cmd;
			struct sli4_sge *sgl;
			struct fcp_cmnd *fcp_cmnd;
			uint32_t *ptr;

			/* 128 byte wqe support here */

			lpfc_cmd = iocbq->context1;
			sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
			fcp_cmnd = lpfc_cmd->fcp_cmnd;

			/* Word 0-2 - FCP_CMND */
			wqe->generic.bde.tus.f.bdeFlags =
				BUFF_TYPE_BDE_IMMED;
			wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
			wqe->generic.bde.addrHigh = 0;
			wqe->generic.bde.addrLow =  88;  /* Word 22 */

			bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
			bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);

			/* Word 22-29  FCP CMND Payload */
			ptr = &wqe->words[22];
			memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
		}
		break;
	case CMD_FCP_ICMND64_CR:
		/* word3 iocb=iotag wqe=payload_offset_len */
		/* Add the FCP_CMD and FCP_RSP sizes to get the offset */
		bf_set(payload_offset_len, &wqe->fcp_icmd,
		       xmit_len + sizeof(struct fcp_rsp));
		bf_set(cmd_buff_len, &wqe->fcp_icmd,
		       0);
		/* word3 iocb=IO_TAG wqe=reserved */
		bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
		/* Always open the exchange */
		bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 1);
		bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE);
		bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
		bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com,
		       LPFC_WQE_LENLOC_NONE);
		bf_set(wqe_erp, &wqe->fcp_icmd.wqe_com,
		       iocbq->iocb.ulpFCP2Rcvy);
		if (iocbq->iocb_flag & LPFC_IO_OAS) {
			bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1);
			bf_set(wqe_ccpe, &wqe->fcp_icmd.wqe_com, 1);
			if (iocbq->priority) {
				bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com,
				       (iocbq->priority << 1));
			} else {
				bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com,
				       (phba->cfg_XLanePriority << 1));
			}
		}
		/* Note, word 10 is already initialized to 0 */

		if (phba->fcp_embed_io) {
			struct lpfc_io_buf *lpfc_cmd;
			struct sli4_sge *sgl;
			struct fcp_cmnd *fcp_cmnd;
			uint32_t *ptr;

			/* 128 byte wqe support here */

			lpfc_cmd = iocbq->context1;
			sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
			fcp_cmnd = lpfc_cmd->fcp_cmnd;

			/* Word 0-2 - FCP_CMND */
			wqe->generic.bde.tus.f.bdeFlags =
				BUFF_TYPE_BDE_IMMED;
			wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
			wqe->generic.bde.addrHigh = 0;
			wqe->generic.bde.addrLow =  88;  /* Word 22 */

			bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
			bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);

			/* Word 22-29  FCP CMND Payload */
			ptr = &wqe->words[22];
			memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
		}
		break;
	case CMD_GEN_REQUEST64_CR:
		/* For this command calculate the xmit length of the
		 * request bde.
		 */
		xmit_len = 0;
		numBdes = iocbq->iocb.un.genreq64.bdl.bdeSize /
			sizeof(struct ulp_bde64);
		for (i = 0; i < numBdes; i++) {
			bde.tus.w = le32_to_cpu(bpl[i].tus.w);
			if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
				break;
			xmit_len += bde.tus.f.bdeSize;
		}
		/* word3 iocb=IO_TAG wqe=request_payload_len */
		wqe->gen_req.request_payload_len = xmit_len;
		/* word4 iocb=parameter wqe=relative_offset memcpy */
		/* word5 [rctl, type, df_ctl, la] copied in memcpy */
		/* word6 context tag copied in memcpy */
		if (iocbq->iocb.ulpCt_h  || iocbq->iocb.ulpCt_l) {
			ct = ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l);
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"2015 Invalid CT %x command 0x%x\n",
				ct, iocbq->iocb.ulpCommand);
			return IOCB_ERROR;
		}
		bf_set(wqe_ct, &wqe->gen_req.wqe_com, 0);
		bf_set(wqe_tmo, &wqe->gen_req.wqe_com, iocbq->iocb.ulpTimeout);
		bf_set(wqe_pu, &wqe->gen_req.wqe_com, iocbq->iocb.ulpPU);
		bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
		bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
		bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
		bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
		bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
		wqe->gen_req.max_response_payload_len = total_len - xmit_len;
		command_type = OTHER_COMMAND;
		break;
	case CMD_XMIT_ELS_RSP64_CX:
		ndlp = (struct lpfc_nodelist *)iocbq->context1;
		/* words0-2 BDE memcpy */
		/* word3 iocb=iotag32 wqe=response_payload_len */
		wqe->xmit_els_rsp.response_payload_len = xmit_len;
		/* word4 */
		wqe->xmit_els_rsp.word4 = 0;
		/* word5 iocb=rsvd wge=did */
		bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest,
			 iocbq->iocb.un.xseq64.xmit_els_remoteID);

		if_type = bf_get(lpfc_sli_intf_if_type,
					&phba->sli4_hba.sli_intf);
		if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
			if (iocbq->vport->fc_flag & FC_PT2PT) {
				bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1);
				bf_set(els_rsp64_sid, &wqe->xmit_els_rsp,
					iocbq->vport->fc_myDID);
				if (iocbq->vport->fc_myDID == Fabric_DID) {
					bf_set(wqe_els_did,
						&wqe->xmit_els_rsp.wqe_dest, 0);
				}
			}
		}
		bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com,
		       ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l));
		bf_set(wqe_pu, &wqe->xmit_els_rsp.wqe_com, iocbq->iocb.ulpPU);
		bf_set(wqe_rcvoxid, &wqe->xmit_els_rsp.wqe_com,
		       iocbq->iocb.unsli3.rcvsli3.ox_id);
		if (!iocbq->iocb.ulpCt_h && iocbq->iocb.ulpCt_l)
			bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com,
			       phba->vpi_ids[iocbq->vport->vpi]);
		bf_set(wqe_dbde, &wqe->xmit_els_rsp.wqe_com, 1);
		bf_set(wqe_iod, &wqe->xmit_els_rsp.wqe_com, LPFC_WQE_IOD_WRITE);
		bf_set(wqe_qosd, &wqe->xmit_els_rsp.wqe_com, 1);
		bf_set(wqe_lenloc, &wqe->xmit_els_rsp.wqe_com,
		       LPFC_WQE_LENLOC_WORD3);
		bf_set(wqe_ebde_cnt, &wqe->xmit_els_rsp.wqe_com, 0);
		bf_set(wqe_rsp_temp_rpi, &wqe->xmit_els_rsp,
		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
		pcmd = (uint32_t *) (((struct lpfc_dmabuf *)
					iocbq->context2)->virt);
		if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) {
				bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1);
				bf_set(els_rsp64_sid, &wqe->xmit_els_rsp,
					iocbq->vport->fc_myDID);
				bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, 1);
				bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com,
					phba->vpi_ids[phba->pport->vpi]);
		}
		command_type = OTHER_COMMAND;
		break;
	case CMD_CLOSE_XRI_CN:
	case CMD_ABORT_XRI_CN:
	case CMD_ABORT_XRI_CX:
		/* words 0-2 memcpy should be 0 rserved */
		/* port will send abts */
		abrt_iotag = iocbq->iocb.un.acxri.abortContextTag;
		if (abrt_iotag != 0 && abrt_iotag <= phba->sli.last_iotag) {
			abrtiocbq = phba->sli.iocbq_lookup[abrt_iotag];
			fip = abrtiocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK;
		} else
			fip = 0;

		if ((iocbq->iocb.ulpCommand == CMD_CLOSE_XRI_CN) || fip)
			/*
			 * The link is down, or the command was ELS_FIP
			 * so the fw does not need to send abts
			 * on the wire.
			 */
			bf_set(abort_cmd_ia, &wqe->abort_cmd, 1);
		else
			bf_set(abort_cmd_ia, &wqe->abort_cmd, 0);
		bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG);
		/* word5 iocb=CONTEXT_TAG|IO_TAG wqe=reserved */
		wqe->abort_cmd.rsrvd5 = 0;
		bf_set(wqe_ct, &wqe->abort_cmd.wqe_com,
			((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l));
		abort_tag = iocbq->iocb.un.acxri.abortIoTag;
		/*
		 * The abort handler will send us CMD_ABORT_XRI_CN or
		 * CMD_CLOSE_XRI_CN and the fw only accepts CMD_ABORT_XRI_CX
		 */
		bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
		bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1);
		bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com,
		       LPFC_WQE_LENLOC_NONE);
		cmnd = CMD_ABORT_XRI_CX;
		command_type = OTHER_COMMAND;
		xritag = 0;
		break;
	case CMD_XMIT_BLS_RSP64_CX:
		ndlp = (struct lpfc_nodelist *)iocbq->context1;
		/* As BLS ABTS RSP WQE is very different from other WQEs,
		 * we re-construct this WQE here based on information in
		 * iocbq from scratch.
		 */
		memset(wqe, 0, sizeof(union lpfc_wqe));
		/* OX_ID is invariable to who sent ABTS to CT exchange */
		bf_set(xmit_bls_rsp64_oxid, &wqe->xmit_bls_rsp,
		       bf_get(lpfc_abts_oxid, &iocbq->iocb.un.bls_rsp));
		if (bf_get(lpfc_abts_orig, &iocbq->iocb.un.bls_rsp) ==
		    LPFC_ABTS_UNSOL_INT) {
			/* ABTS sent by initiator to CT exchange, the
			 * RX_ID field will be filled with the newly
			 * allocated responder XRI.
			 */
			bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp,
			       iocbq->sli4_xritag);
		} else {
			/* ABTS sent by responder to CT exchange, the
			 * RX_ID field will be filled with the responder
			 * RX_ID from ABTS.
			 */
			bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp,
			       bf_get(lpfc_abts_rxid, &iocbq->iocb.un.bls_rsp));
		}
		bf_set(xmit_bls_rsp64_seqcnthi, &wqe->xmit_bls_rsp, 0xffff);
		bf_set(wqe_xmit_bls_pt, &wqe->xmit_bls_rsp.wqe_dest, 0x1);

		/* Use CT=VPI */
		bf_set(wqe_els_did, &wqe->xmit_bls_rsp.wqe_dest,
			ndlp->nlp_DID);
		bf_set(xmit_bls_rsp64_temprpi, &wqe->xmit_bls_rsp,
			iocbq->iocb.ulpContext);
		bf_set(wqe_ct, &wqe->xmit_bls_rsp.wqe_com, 1);
		bf_set(wqe_ctxt_tag, &wqe->xmit_bls_rsp.wqe_com,
			phba->vpi_ids[phba->pport->vpi]);
		bf_set(wqe_qosd, &wqe->xmit_bls_rsp.wqe_com, 1);
		bf_set(wqe_lenloc, &wqe->xmit_bls_rsp.wqe_com,
		       LPFC_WQE_LENLOC_NONE);
		/* Overwrite the pre-set comnd type with OTHER_COMMAND */
		command_type = OTHER_COMMAND;
		if (iocbq->iocb.un.xseq64.w5.hcsw.Rctl == FC_RCTL_BA_RJT) {
			bf_set(xmit_bls_rsp64_rjt_vspec, &wqe->xmit_bls_rsp,
			       bf_get(lpfc_vndr_code, &iocbq->iocb.un.bls_rsp));
			bf_set(xmit_bls_rsp64_rjt_expc, &wqe->xmit_bls_rsp,
			       bf_get(lpfc_rsn_expln, &iocbq->iocb.un.bls_rsp));
			bf_set(xmit_bls_rsp64_rjt_rsnc, &wqe->xmit_bls_rsp,
			       bf_get(lpfc_rsn_code, &iocbq->iocb.un.bls_rsp));
		}

		break;
	case CMD_SEND_FRAME:
		bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag);
		bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag);
		return 0;
	case CMD_XRI_ABORTED_CX:
	case CMD_CREATE_XRI_CR: /* Do we expect to use this? */
	case CMD_IOCB_FCP_IBIDIR64_CR: /* bidirectional xfer */
	case CMD_FCP_TSEND64_CX: /* Target mode send xfer-ready */
	case CMD_FCP_TRSP64_CX: /* Target mode rcv */
	case CMD_FCP_AUTO_TRSP_CX: /* Auto target rsp */
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"2014 Invalid command 0x%x\n",
				iocbq->iocb.ulpCommand);
		return IOCB_ERROR;
		break;
	}

	if (iocbq->iocb_flag & LPFC_IO_DIF_PASS)
		bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_PASSTHRU);
	else if (iocbq->iocb_flag & LPFC_IO_DIF_STRIP)
		bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_STRIP);
	else if (iocbq->iocb_flag & LPFC_IO_DIF_INSERT)
		bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_INSERT);
	iocbq->iocb_flag &= ~(LPFC_IO_DIF_PASS | LPFC_IO_DIF_STRIP |
			      LPFC_IO_DIF_INSERT);
	bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag);
	bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag);
	wqe->generic.wqe_com.abort_tag = abort_tag;
	bf_set(wqe_cmd_type, &wqe->generic.wqe_com, command_type);
	bf_set(wqe_cmnd, &wqe->generic.wqe_com, cmnd);
	bf_set(wqe_class, &wqe->generic.wqe_com, iocbq->iocb.ulpClass);
	bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
	return 0;
}

/**
 * __lpfc_sli_issue_iocb_s4 - SLI4 device lockless ver of lpfc_sli_issue_iocb
 * @phba: Pointer to HBA context object.
 * @ring_number: SLI ring number to issue iocb on.
 * @piocb: Pointer to command iocb.
 * @flag: Flag indicating if this command can be put into txq.
 *
 * __lpfc_sli_issue_iocb_s4 is used by other functions in the driver to issue
 * an iocb command to an HBA with SLI-4 interface spec.
 *
 * This function is called with hbalock held. The function will return success
 * after it successfully submit the iocb to firmware or after adding to the
 * txq.
 **/
static int
__lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number,
			 struct lpfc_iocbq *piocb, uint32_t flag)
{
	struct lpfc_sglq *sglq;
	union lpfc_wqe128 wqe;
	struct lpfc_queue *wq;
	struct lpfc_sli_ring *pring;

	/* Get the WQ */
	if ((piocb->iocb_flag & LPFC_IO_FCP) ||
	    (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) {
		wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq;
	} else {
		wq = phba->sli4_hba.els_wq;
	}

	/* Get corresponding ring */
	pring = wq->pring;

	/*
	 * The WQE can be either 64 or 128 bytes,
	 */

	lockdep_assert_held(&pring->ring_lock);

	if (piocb->sli4_xritag == NO_XRI) {
		if (piocb->iocb.ulpCommand == CMD_ABORT_XRI_CN ||
		    piocb->iocb.ulpCommand == CMD_CLOSE_XRI_CN)
			sglq = NULL;
		else {
			if (!list_empty(&pring->txq)) {
				if (!(flag & SLI_IOCB_RET_IOCB)) {
					__lpfc_sli_ringtx_put(phba,
						pring, piocb);
					return IOCB_SUCCESS;
				} else {
					return IOCB_BUSY;
				}
			} else {
				sglq = __lpfc_sli_get_els_sglq(phba, piocb);
				if (!sglq) {
					if (!(flag & SLI_IOCB_RET_IOCB)) {
						__lpfc_sli_ringtx_put(phba,
								pring,
								piocb);
						return IOCB_SUCCESS;
					} else
						return IOCB_BUSY;
				}
			}
		}
	} else if (piocb->iocb_flag &  LPFC_IO_FCP)
		/* These IO's already have an XRI and a mapped sgl. */
		sglq = NULL;
	else {
		/*
		 * This is a continuation of a commandi,(CX) so this
		 * sglq is on the active list
		 */
		sglq = __lpfc_get_active_sglq(phba, piocb->sli4_lxritag);
		if (!sglq)
			return IOCB_ERROR;
	}

	if (sglq) {
		piocb->sli4_lxritag = sglq->sli4_lxritag;
		piocb->sli4_xritag = sglq->sli4_xritag;
		if (NO_XRI == lpfc_sli4_bpl2sgl(phba, piocb, sglq))
			return IOCB_ERROR;
	}

	if (lpfc_sli4_iocb2wqe(phba, piocb, &wqe))
		return IOCB_ERROR;

	if (lpfc_sli4_wq_put(wq, &wqe))
		return IOCB_ERROR;
	lpfc_sli_ringtxcmpl_put(phba, pring, piocb);

	return 0;
}

/**
 * __lpfc_sli_issue_iocb - Wrapper func of lockless version for issuing iocb
 *
 * This routine wraps the actual lockless version for issusing IOCB function
 * pointer from the lpfc_hba struct.
 *
 * Return codes:
 * IOCB_ERROR - Error
 * IOCB_SUCCESS - Success
 * IOCB_BUSY - Busy
 **/
int
__lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
		struct lpfc_iocbq *piocb, uint32_t flag)
{
	return phba->__lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
}

/**
 * lpfc_sli_api_table_setup - Set up sli api function jump table
 * @phba: The hba struct for which this call is being executed.
 * @dev_grp: The HBA PCI-Device group number.
 *
 * This routine sets up the SLI interface API function jump table in @phba
 * struct.
 * Returns: 0 - success, -ENODEV - failure.
 **/
int
lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
{

	switch (dev_grp) {
	case LPFC_PCI_DEV_LP:
		phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s3;
		phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s3;
		break;
	case LPFC_PCI_DEV_OC:
		phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s4;
		phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s4;
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"1419 Invalid HBA PCI-device group: 0x%x\n",
				dev_grp);
		return -ENODEV;
		break;
	}
	phba->lpfc_get_iocb_from_iocbq = lpfc_get_iocb_from_iocbq;
	return 0;
}

/**
 * lpfc_sli4_calc_ring - Calculates which ring to use
 * @phba: Pointer to HBA context object.
 * @piocb: Pointer to command iocb.
 *
 * For SLI4 only, FCP IO can deferred to one fo many WQs, based on
 * hba_wqidx, thus we need to calculate the corresponding ring.
 * Since ABORTS must go on the same WQ of the command they are
 * aborting, we use command's hba_wqidx.
 */
struct lpfc_sli_ring *
lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb)
{
	struct lpfc_io_buf *lpfc_cmd;

	if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) {
		if (unlikely(!phba->sli4_hba.hdwq))
			return NULL;
		/*
		 * for abort iocb hba_wqidx should already
		 * be setup based on what work queue we used.
		 */
		if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) {
			lpfc_cmd = (struct lpfc_io_buf *)piocb->context1;
			piocb->hba_wqidx = lpfc_cmd->hdwq_no;
		}
		return phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq->pring;
	} else {
		if (unlikely(!phba->sli4_hba.els_wq))
			return NULL;
		piocb->hba_wqidx = 0;
		return phba->sli4_hba.els_wq->pring;
	}
}

/**
 * lpfc_sli_issue_iocb - Wrapper function for __lpfc_sli_issue_iocb
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @piocb: Pointer to command iocb.
 * @flag: Flag indicating if this command can be put into txq.
 *
 * lpfc_sli_issue_iocb is a wrapper around __lpfc_sli_issue_iocb
 * function. This function gets the hbalock and calls
 * __lpfc_sli_issue_iocb function and will return the error returned
 * by __lpfc_sli_issue_iocb function. This wrapper is used by
 * functions which do not hold hbalock.
 **/
int
lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
		    struct lpfc_iocbq *piocb, uint32_t flag)
{
	struct lpfc_sli_ring *pring;
	unsigned long iflags;
	int rc;

	if (phba->sli_rev == LPFC_SLI_REV4) {
		pring = lpfc_sli4_calc_ring(phba, piocb);
		if (unlikely(pring == NULL))
			return IOCB_ERROR;

		spin_lock_irqsave(&pring->ring_lock, iflags);
		rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
		spin_unlock_irqrestore(&pring->ring_lock, iflags);
	} else {
		/* For now, SLI2/3 will still use hbalock */
		spin_lock_irqsave(&phba->hbalock, iflags);
		rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
		spin_unlock_irqrestore(&phba->hbalock, iflags);
	}
	return rc;
}

/**
 * lpfc_extra_ring_setup - Extra ring setup function
 * @phba: Pointer to HBA context object.
 *
 * This function is called while driver attaches with the
 * HBA to setup the extra ring. The extra ring is used
 * only when driver needs to support target mode functionality
 * or IP over FC functionalities.
 *
 * This function is called with no lock held. SLI3 only.
 **/
static int
lpfc_extra_ring_setup( struct lpfc_hba *phba)
{
	struct lpfc_sli *psli;
	struct lpfc_sli_ring *pring;

	psli = &phba->sli;

	/* Adjust cmd/rsp ring iocb entries more evenly */

	/* Take some away from the FCP ring */
	pring = &psli->sli3_ring[LPFC_FCP_RING];
	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R1XTRA_ENTRIES;
	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R1XTRA_ENTRIES;
	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R3XTRA_ENTRIES;
	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R3XTRA_ENTRIES;

	/* and give them to the extra ring */
	pring = &psli->sli3_ring[LPFC_EXTRA_RING];

	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R1XTRA_ENTRIES;
	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R1XTRA_ENTRIES;
	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R3XTRA_ENTRIES;
	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R3XTRA_ENTRIES;

	/* Setup default profile for this ring */
	pring->iotag_max = 4096;
	pring->num_mask = 1;
	pring->prt[0].profile = 0;      /* Mask 0 */
	pring->prt[0].rctl = phba->cfg_multi_ring_rctl;
	pring->prt[0].type = phba->cfg_multi_ring_type;
	pring->prt[0].lpfc_sli_rcv_unsol_event = NULL;
	return 0;
}

/* lpfc_sli_abts_err_handler - handle a failed ABTS request from an SLI3 port.
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to iocb object.
 *
 * The async_event handler calls this routine when it receives
 * an ASYNC_STATUS_CN event from the port.  The port generates
 * this event when an Abort Sequence request to an rport fails
 * twice in succession.  The abort could be originated by the
 * driver or by the port.  The ABTS could have been for an ELS
 * or FCP IO.  The port only generates this event when an ABTS
 * fails to complete after one retry.
 */
static void
lpfc_sli_abts_err_handler(struct lpfc_hba *phba,
			  struct lpfc_iocbq *iocbq)
{
	struct lpfc_nodelist *ndlp = NULL;
	uint16_t rpi = 0, vpi = 0;
	struct lpfc_vport *vport = NULL;

	/* The rpi in the ulpContext is vport-sensitive. */
	vpi = iocbq->iocb.un.asyncstat.sub_ctxt_tag;
	rpi = iocbq->iocb.ulpContext;

	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
			"3092 Port generated ABTS async event "
			"on vpi %d rpi %d status 0x%x\n",
			vpi, rpi, iocbq->iocb.ulpStatus);

	vport = lpfc_find_vport_by_vpid(phba, vpi);
	if (!vport)
		goto err_exit;
	ndlp = lpfc_findnode_rpi(vport, rpi);
	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp))
		goto err_exit;

	if (iocbq->iocb.ulpStatus == IOSTAT_LOCAL_REJECT)
		lpfc_sli_abts_recover_port(vport, ndlp);
	return;

 err_exit:
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"3095 Event Context not found, no "
			"action on vpi %d rpi %d status 0x%x, reason 0x%x\n",
			iocbq->iocb.ulpContext, iocbq->iocb.ulpStatus,
			vpi, rpi);
}

/* lpfc_sli4_abts_err_handler - handle a failed ABTS request from an SLI4 port.
 * @phba: pointer to HBA context object.
 * @ndlp: nodelist pointer for the impacted rport.
 * @axri: pointer to the wcqe containing the failed exchange.
 *
 * The driver calls this routine when it receives an ABORT_XRI_FCP CQE from the
 * port.  The port generates this event when an abort exchange request to an
 * rport fails twice in succession with no reply.  The abort could be originated
 * by the driver or by the port.  The ABTS could have been for an ELS or FCP IO.
 */
void
lpfc_sli4_abts_err_handler(struct lpfc_hba *phba,
			   struct lpfc_nodelist *ndlp,
			   struct sli4_wcqe_xri_aborted *axri)
{
	struct lpfc_vport *vport;
	uint32_t ext_status = 0;

	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"3115 Node Context not found, driver "
				"ignoring abts err event\n");
		return;
	}

	vport = ndlp->vport;
	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
			"3116 Port generated FCP XRI ABORT event on "
			"vpi %d rpi %d xri x%x status 0x%x parameter x%x\n",
			ndlp->vport->vpi, phba->sli4_hba.rpi_ids[ndlp->nlp_rpi],
			bf_get(lpfc_wcqe_xa_xri, axri),
			bf_get(lpfc_wcqe_xa_status, axri),
			axri->parameter);

	/*
	 * Catch the ABTS protocol failure case.  Older OCe FW releases returned
	 * LOCAL_REJECT and 0 for a failed ABTS exchange and later OCe and
	 * LPe FW releases returned LOCAL_REJECT and SEQUENCE_TIMEOUT.
	 */
	ext_status = axri->parameter & IOERR_PARAM_MASK;
	if ((bf_get(lpfc_wcqe_xa_status, axri) == IOSTAT_LOCAL_REJECT) &&
	    ((ext_status == IOERR_SEQUENCE_TIMEOUT) || (ext_status == 0)))
		lpfc_sli_abts_recover_port(vport, ndlp);
}

/**
 * lpfc_sli_async_event_handler - ASYNC iocb handler function
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @iocbq: Pointer to iocb object.
 *
 * This function is called by the slow ring event handler
 * function when there is an ASYNC event iocb in the ring.
 * This function is called with no lock held.
 * Currently this function handles only temperature related
 * ASYNC events. The function decodes the temperature sensor
 * event message and posts events for the management applications.
 **/
static void
lpfc_sli_async_event_handler(struct lpfc_hba * phba,
	struct lpfc_sli_ring * pring, struct lpfc_iocbq * iocbq)
{
	IOCB_t *icmd;
	uint16_t evt_code;
	struct temp_event temp_event_data;
	struct Scsi_Host *shost;
	uint32_t *iocb_w;

	icmd = &iocbq->iocb;
	evt_code = icmd->un.asyncstat.evt_code;

	switch (evt_code) {
	case ASYNC_TEMP_WARN:
	case ASYNC_TEMP_SAFE:
		temp_event_data.data = (uint32_t) icmd->ulpContext;
		temp_event_data.event_type = FC_REG_TEMPERATURE_EVENT;
		if (evt_code == ASYNC_TEMP_WARN) {
			temp_event_data.event_code = LPFC_THRESHOLD_TEMP;
			lpfc_printf_log(phba, KERN_ERR, LOG_TEMP,
				"0347 Adapter is very hot, please take "
				"corrective action. temperature : %d Celsius\n",
				(uint32_t) icmd->ulpContext);
		} else {
			temp_event_data.event_code = LPFC_NORMAL_TEMP;
			lpfc_printf_log(phba, KERN_ERR, LOG_TEMP,
				"0340 Adapter temperature is OK now. "
				"temperature : %d Celsius\n",
				(uint32_t) icmd->ulpContext);
		}

		/* Send temperature change event to applications */
		shost = lpfc_shost_from_vport(phba->pport);
		fc_host_post_vendor_event(shost, fc_get_event_number(),
			sizeof(temp_event_data), (char *) &temp_event_data,
			LPFC_NL_VENDOR_ID);
		break;
	case ASYNC_STATUS_CN:
		lpfc_sli_abts_err_handler(phba, iocbq);
		break;
	default:
		iocb_w = (uint32_t *) icmd;
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
			"0346 Ring %d handler: unexpected ASYNC_STATUS"
			" evt_code 0x%x\n"
			"W0  0x%08x W1  0x%08x W2  0x%08x W3  0x%08x\n"
			"W4  0x%08x W5  0x%08x W6  0x%08x W7  0x%08x\n"
			"W8  0x%08x W9  0x%08x W10 0x%08x W11 0x%08x\n"
			"W12 0x%08x W13 0x%08x W14 0x%08x W15 0x%08x\n",
			pring->ringno, icmd->un.asyncstat.evt_code,
			iocb_w[0], iocb_w[1], iocb_w[2], iocb_w[3],
			iocb_w[4], iocb_w[5], iocb_w[6], iocb_w[7],
			iocb_w[8], iocb_w[9], iocb_w[10], iocb_w[11],
			iocb_w[12], iocb_w[13], iocb_w[14], iocb_w[15]);

		break;
	}
}


/**
 * lpfc_sli4_setup - SLI ring setup function
 * @phba: Pointer to HBA context object.
 *
 * lpfc_sli_setup sets up rings of the SLI interface with
 * number of iocbs per ring and iotags. This function is
 * called while driver attach to the HBA and before the
 * interrupts are enabled. So there is no need for locking.
 *
 * This function always returns 0.
 **/
int
lpfc_sli4_setup(struct lpfc_hba *phba)
{
	struct lpfc_sli_ring *pring;

	pring = phba->sli4_hba.els_wq->pring;
	pring->num_mask = LPFC_MAX_RING_MASK;
	pring->prt[0].profile = 0;	/* Mask 0 */
	pring->prt[0].rctl = FC_RCTL_ELS_REQ;
	pring->prt[0].type = FC_TYPE_ELS;
	pring->prt[0].lpfc_sli_rcv_unsol_event =
	    lpfc_els_unsol_event;
	pring->prt[1].profile = 0;	/* Mask 1 */
	pring->prt[1].rctl = FC_RCTL_ELS_REP;
	pring->prt[1].type = FC_TYPE_ELS;
	pring->prt[1].lpfc_sli_rcv_unsol_event =
	    lpfc_els_unsol_event;
	pring->prt[2].profile = 0;	/* Mask 2 */
	/* NameServer Inquiry */
	pring->prt[2].rctl = FC_RCTL_DD_UNSOL_CTL;
	/* NameServer */
	pring->prt[2].type = FC_TYPE_CT;
	pring->prt[2].lpfc_sli_rcv_unsol_event =
	    lpfc_ct_unsol_event;
	pring->prt[3].profile = 0;	/* Mask 3 */
	/* NameServer response */
	pring->prt[3].rctl = FC_RCTL_DD_SOL_CTL;
	/* NameServer */
	pring->prt[3].type = FC_TYPE_CT;
	pring->prt[3].lpfc_sli_rcv_unsol_event =
	    lpfc_ct_unsol_event;
	return 0;
}

/**
 * lpfc_sli_setup - SLI ring setup function
 * @phba: Pointer to HBA context object.
 *
 * lpfc_sli_setup sets up rings of the SLI interface with
 * number of iocbs per ring and iotags. This function is
 * called while driver attach to the HBA and before the
 * interrupts are enabled. So there is no need for locking.
 *
 * This function always returns 0. SLI3 only.
 **/
int
lpfc_sli_setup(struct lpfc_hba *phba)
{
	int i, totiocbsize = 0;
	struct lpfc_sli *psli = &phba->sli;
	struct lpfc_sli_ring *pring;

	psli->num_rings = MAX_SLI3_CONFIGURED_RINGS;
	psli->sli_flag = 0;

	psli->iocbq_lookup = NULL;
	psli->iocbq_lookup_len = 0;
	psli->last_iotag = 0;

	for (i = 0; i < psli->num_rings; i++) {
		pring = &psli->sli3_ring[i];
		switch (i) {
		case LPFC_FCP_RING:	/* ring 0 - FCP */
			/* numCiocb and numRiocb are used in config_port */
			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R0_ENTRIES;
			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R0_ENTRIES;
			pring->sli.sli3.numCiocb +=
				SLI2_IOCB_CMD_R1XTRA_ENTRIES;
			pring->sli.sli3.numRiocb +=
				SLI2_IOCB_RSP_R1XTRA_ENTRIES;
			pring->sli.sli3.numCiocb +=
				SLI2_IOCB_CMD_R3XTRA_ENTRIES;
			pring->sli.sli3.numRiocb +=
				SLI2_IOCB_RSP_R3XTRA_ENTRIES;
			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
							SLI3_IOCB_CMD_SIZE :
							SLI2_IOCB_CMD_SIZE;
			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
							SLI3_IOCB_RSP_SIZE :
							SLI2_IOCB_RSP_SIZE;
			pring->iotag_ctr = 0;
			pring->iotag_max =
			    (phba->cfg_hba_queue_depth * 2);
			pring->fast_iotag = pring->iotag_max;
			pring->num_mask = 0;
			break;
		case LPFC_EXTRA_RING:	/* ring 1 - EXTRA */
			/* numCiocb and numRiocb are used in config_port */
			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R1_ENTRIES;
			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R1_ENTRIES;
			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
							SLI3_IOCB_CMD_SIZE :
							SLI2_IOCB_CMD_SIZE;
			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
							SLI3_IOCB_RSP_SIZE :
							SLI2_IOCB_RSP_SIZE;
			pring->iotag_max = phba->cfg_hba_queue_depth;
			pring->num_mask = 0;
			break;
		case LPFC_ELS_RING:	/* ring 2 - ELS / CT */
			/* numCiocb and numRiocb are used in config_port */
			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R2_ENTRIES;
			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R2_ENTRIES;
			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
							SLI3_IOCB_CMD_SIZE :
							SLI2_IOCB_CMD_SIZE;
			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
							SLI3_IOCB_RSP_SIZE :
							SLI2_IOCB_RSP_SIZE;
			pring->fast_iotag = 0;
			pring->iotag_ctr = 0;
			pring->iotag_max = 4096;
			pring->lpfc_sli_rcv_async_status =
				lpfc_sli_async_event_handler;
			pring->num_mask = LPFC_MAX_RING_MASK;
			pring->prt[0].profile = 0;	/* Mask 0 */
			pring->prt[0].rctl = FC_RCTL_ELS_REQ;
			pring->prt[0].type = FC_TYPE_ELS;
			pring->prt[0].lpfc_sli_rcv_unsol_event =
			    lpfc_els_unsol_event;
			pring->prt[1].profile = 0;	/* Mask 1 */
			pring->prt[1].rctl = FC_RCTL_ELS_REP;
			pring->prt[1].type = FC_TYPE_ELS;
			pring->prt[1].lpfc_sli_rcv_unsol_event =
			    lpfc_els_unsol_event;
			pring->prt[2].profile = 0;	/* Mask 2 */
			/* NameServer Inquiry */
			pring->prt[2].rctl = FC_RCTL_DD_UNSOL_CTL;
			/* NameServer */
			pring->prt[2].type = FC_TYPE_CT;
			pring->prt[2].lpfc_sli_rcv_unsol_event =
			    lpfc_ct_unsol_event;
			pring->prt[3].profile = 0;	/* Mask 3 */
			/* NameServer response */
			pring->prt[3].rctl = FC_RCTL_DD_SOL_CTL;
			/* NameServer */
			pring->prt[3].type = FC_TYPE_CT;
			pring->prt[3].lpfc_sli_rcv_unsol_event =
			    lpfc_ct_unsol_event;
			break;
		}
		totiocbsize += (pring->sli.sli3.numCiocb *
			pring->sli.sli3.sizeCiocb) +
			(pring->sli.sli3.numRiocb * pring->sli.sli3.sizeRiocb);
	}
	if (totiocbsize > MAX_SLIM_IOCB_SIZE) {
		/* Too many cmd / rsp ring entries in SLI2 SLIM */
		printk(KERN_ERR "%d:0462 Too many cmd / rsp ring entries in "
		       "SLI2 SLIM Data: x%x x%lx\n",
		       phba->brd_no, totiocbsize,
		       (unsigned long) MAX_SLIM_IOCB_SIZE);
	}
	if (phba->cfg_multi_ring_support == 2)
		lpfc_extra_ring_setup(phba);

	return 0;
}

/**
 * lpfc_sli4_queue_init - Queue initialization function
 * @phba: Pointer to HBA context object.
 *
 * lpfc_sli4_queue_init sets up mailbox queues and iocb queues for each
 * ring. This function also initializes ring indices of each ring.
 * This function is called during the initialization of the SLI
 * interface of an HBA.
 * This function is called with no lock held and always returns
 * 1.
 **/
void
lpfc_sli4_queue_init(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli;
	struct lpfc_sli_ring *pring;
	int i;

	psli = &phba->sli;
	spin_lock_irq(&phba->hbalock);
	INIT_LIST_HEAD(&psli->mboxq);
	INIT_LIST_HEAD(&psli->mboxq_cmpl);
	/* Initialize list headers for txq and txcmplq as double linked lists */
	for (i = 0; i < phba->cfg_hdw_queue; i++) {
		pring = phba->sli4_hba.hdwq[i].fcp_wq->pring;
		pring->flag = 0;
		pring->ringno = LPFC_FCP_RING;
		pring->txcmplq_cnt = 0;
		INIT_LIST_HEAD(&pring->txq);
		INIT_LIST_HEAD(&pring->txcmplq);
		INIT_LIST_HEAD(&pring->iocb_continueq);
		spin_lock_init(&pring->ring_lock);
	}
	pring = phba->sli4_hba.els_wq->pring;
	pring->flag = 0;
	pring->ringno = LPFC_ELS_RING;
	pring->txcmplq_cnt = 0;
	INIT_LIST_HEAD(&pring->txq);
	INIT_LIST_HEAD(&pring->txcmplq);
	INIT_LIST_HEAD(&pring->iocb_continueq);
	spin_lock_init(&pring->ring_lock);

	if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
		for (i = 0; i < phba->cfg_hdw_queue; i++) {
			pring = phba->sli4_hba.hdwq[i].nvme_wq->pring;
			pring->flag = 0;
			pring->ringno = LPFC_FCP_RING;
			pring->txcmplq_cnt = 0;
			INIT_LIST_HEAD(&pring->txq);
			INIT_LIST_HEAD(&pring->txcmplq);
			INIT_LIST_HEAD(&pring->iocb_continueq);
			spin_lock_init(&pring->ring_lock);
		}
		pring = phba->sli4_hba.nvmels_wq->pring;
		pring->flag = 0;
		pring->ringno = LPFC_ELS_RING;
		pring->txcmplq_cnt = 0;
		INIT_LIST_HEAD(&pring->txq);
		INIT_LIST_HEAD(&pring->txcmplq);
		INIT_LIST_HEAD(&pring->iocb_continueq);
		spin_lock_init(&pring->ring_lock);
	}

	spin_unlock_irq(&phba->hbalock);
}

/**
 * lpfc_sli_queue_init - Queue initialization function
 * @phba: Pointer to HBA context object.
 *
 * lpfc_sli_queue_init sets up mailbox queues and iocb queues for each
 * ring. This function also initializes ring indices of each ring.
 * This function is called during the initialization of the SLI
 * interface of an HBA.
 * This function is called with no lock held and always returns
 * 1.
 **/
void
lpfc_sli_queue_init(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli;
	struct lpfc_sli_ring *pring;
	int i;

	psli = &phba->sli;
	spin_lock_irq(&phba->hbalock);
	INIT_LIST_HEAD(&psli->mboxq);
	INIT_LIST_HEAD(&psli->mboxq_cmpl);
	/* Initialize list headers for txq and txcmplq as double linked lists */
	for (i = 0; i < psli->num_rings; i++) {
		pring = &psli->sli3_ring[i];
		pring->ringno = i;
		pring->sli.sli3.next_cmdidx  = 0;
		pring->sli.sli3.local_getidx = 0;
		pring->sli.sli3.cmdidx = 0;
		INIT_LIST_HEAD(&pring->iocb_continueq);
		INIT_LIST_HEAD(&pring->iocb_continue_saveq);
		INIT_LIST_HEAD(&pring->postbufq);
		pring->flag = 0;
		INIT_LIST_HEAD(&pring->txq);
		INIT_LIST_HEAD(&pring->txcmplq);
		spin_lock_init(&pring->ring_lock);
	}
	spin_unlock_irq(&phba->hbalock);
}

/**
 * lpfc_sli_mbox_sys_flush - Flush mailbox command sub-system
 * @phba: Pointer to HBA context object.
 *
 * This routine flushes the mailbox command subsystem. It will unconditionally
 * flush all the mailbox commands in the three possible stages in the mailbox
 * command sub-system: pending mailbox command queue; the outstanding mailbox
 * command; and completed mailbox command queue. It is caller's responsibility
 * to make sure that the driver is in the proper state to flush the mailbox
 * command sub-system. Namely, the posting of mailbox commands into the
 * pending mailbox command queue from the various clients must be stopped;
 * either the HBA is in a state that it will never works on the outstanding
 * mailbox command (such as in EEH or ERATT conditions) or the outstanding
 * mailbox command has been completed.
 **/
static void
lpfc_sli_mbox_sys_flush(struct lpfc_hba *phba)
{
	LIST_HEAD(completions);
	struct lpfc_sli *psli = &phba->sli;
	LPFC_MBOXQ_t *pmb;
	unsigned long iflag;

	/* Disable softirqs, including timers from obtaining phba->hbalock */
	local_bh_disable();

	/* Flush all the mailbox commands in the mbox system */
	spin_lock_irqsave(&phba->hbalock, iflag);

	/* The pending mailbox command queue */
	list_splice_init(&phba->sli.mboxq, &completions);
	/* The outstanding active mailbox command */
	if (psli->mbox_active) {
		list_add_tail(&psli->mbox_active->list, &completions);
		psli->mbox_active = NULL;
		psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
	}
	/* The completed mailbox command queue */
	list_splice_init(&phba->sli.mboxq_cmpl, &completions);
	spin_unlock_irqrestore(&phba->hbalock, iflag);

	/* Enable softirqs again, done with phba->hbalock */
	local_bh_enable();

	/* Return all flushed mailbox commands with MBX_NOT_FINISHED status */
	while (!list_empty(&completions)) {
		list_remove_head(&completions, pmb, LPFC_MBOXQ_t, list);
		pmb->u.mb.mbxStatus = MBX_NOT_FINISHED;
		if (pmb->mbox_cmpl)
			pmb->mbox_cmpl(phba, pmb);
	}
}

/**
 * lpfc_sli_host_down - Vport cleanup function
 * @vport: Pointer to virtual port object.
 *
 * lpfc_sli_host_down is called to clean up the resources
 * associated with a vport before destroying virtual
 * port data structures.
 * This function does following operations:
 * - Free discovery resources associated with this virtual
 *   port.
 * - Free iocbs associated with this virtual port in
 *   the txq.
 * - Send abort for all iocb commands associated with this
 *   vport in txcmplq.
 *
 * This function is called with no lock held and always returns 1.
 **/
int
lpfc_sli_host_down(struct lpfc_vport *vport)
{
	LIST_HEAD(completions);
	struct lpfc_hba *phba = vport->phba;
	struct lpfc_sli *psli = &phba->sli;
	struct lpfc_queue *qp = NULL;
	struct lpfc_sli_ring *pring;
	struct lpfc_iocbq *iocb, *next_iocb;
	int i;
	unsigned long flags = 0;
	uint16_t prev_pring_flag;

	lpfc_cleanup_discovery_resources(vport);

	spin_lock_irqsave(&phba->hbalock, flags);

	/*
	 * Error everything on the txq since these iocbs
	 * have not been given to the FW yet.
	 * Also issue ABTS for everything on the txcmplq
	 */
	if (phba->sli_rev != LPFC_SLI_REV4) {
		for (i = 0; i < psli->num_rings; i++) {
			pring = &psli->sli3_ring[i];
			prev_pring_flag = pring->flag;
			/* Only slow rings */
			if (pring->ringno == LPFC_ELS_RING) {
				pring->flag |= LPFC_DEFERRED_RING_EVENT;
				/* Set the lpfc data pending flag */
				set_bit(LPFC_DATA_READY, &phba->data_flags);
			}
			list_for_each_entry_safe(iocb, next_iocb,
						 &pring->txq, list) {
				if (iocb->vport != vport)
					continue;
				list_move_tail(&iocb->list, &completions);
			}
			list_for_each_entry_safe(iocb, next_iocb,
						 &pring->txcmplq, list) {
				if (iocb->vport != vport)
					continue;
				lpfc_sli_issue_abort_iotag(phba, pring, iocb);
			}
			pring->flag = prev_pring_flag;
		}
	} else {
		list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
			pring = qp->pring;
			if (!pring)
				continue;
			if (pring == phba->sli4_hba.els_wq->pring) {
				pring->flag |= LPFC_DEFERRED_RING_EVENT;
				/* Set the lpfc data pending flag */
				set_bit(LPFC_DATA_READY, &phba->data_flags);
			}
			prev_pring_flag = pring->flag;
			spin_lock_irq(&pring->ring_lock);
			list_for_each_entry_safe(iocb, next_iocb,
						 &pring->txq, list) {
				if (iocb->vport != vport)
					continue;
				list_move_tail(&iocb->list, &completions);
			}
			spin_unlock_irq(&pring->ring_lock);
			list_for_each_entry_safe(iocb, next_iocb,
						 &pring->txcmplq, list) {
				if (iocb->vport != vport)
					continue;
				lpfc_sli_issue_abort_iotag(phba, pring, iocb);
			}
			pring->flag = prev_pring_flag;
		}
	}
	spin_unlock_irqrestore(&phba->hbalock, flags);

	/* Cancel all the IOCBs from the completions list */
	lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
			      IOERR_SLI_DOWN);
	return 1;
}

/**
 * lpfc_sli_hba_down - Resource cleanup function for the HBA
 * @phba: Pointer to HBA context object.
 *
 * This function cleans up all iocb, buffers, mailbox commands
 * while shutting down the HBA. This function is called with no
 * lock held and always returns 1.
 * This function does the following to cleanup driver resources:
 * - Free discovery resources for each virtual port
 * - Cleanup any pending fabric iocbs
 * - Iterate through the iocb txq and free each entry
 *   in the list.
 * - Free up any buffer posted to the HBA
 * - Free mailbox commands in the mailbox queue.
 **/
int
lpfc_sli_hba_down(struct lpfc_hba *phba)
{
	LIST_HEAD(completions);
	struct lpfc_sli *psli = &phba->sli;
	struct lpfc_queue *qp = NULL;
	struct lpfc_sli_ring *pring;
	struct lpfc_dmabuf *buf_ptr;
	unsigned long flags = 0;
	int i;

	/* Shutdown the mailbox command sub-system */
	lpfc_sli_mbox_sys_shutdown(phba, LPFC_MBX_WAIT);

	lpfc_hba_down_prep(phba);

	/* Disable softirqs, including timers from obtaining phba->hbalock */
	local_bh_disable();

	lpfc_fabric_abort_hba(phba);

	spin_lock_irqsave(&phba->hbalock, flags);

	/*
	 * Error everything on the txq since these iocbs
	 * have not been given to the FW yet.
	 */
	if (phba->sli_rev != LPFC_SLI_REV4) {
		for (i = 0; i < psli->num_rings; i++) {
			pring = &psli->sli3_ring[i];
			/* Only slow rings */
			if (pring->ringno == LPFC_ELS_RING) {
				pring->flag |= LPFC_DEFERRED_RING_EVENT;
				/* Set the lpfc data pending flag */
				set_bit(LPFC_DATA_READY, &phba->data_flags);
			}
			list_splice_init(&pring->txq, &completions);
		}
	} else {
		list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
			pring = qp->pring;
			if (!pring)
				continue;
			spin_lock_irq(&pring->ring_lock);
			list_splice_init(&pring->txq, &completions);
			spin_unlock_irq(&pring->ring_lock);
			if (pring == phba->sli4_hba.els_wq->pring) {
				pring->flag |= LPFC_DEFERRED_RING_EVENT;
				/* Set the lpfc data pending flag */
				set_bit(LPFC_DATA_READY, &phba->data_flags);
			}
		}
	}
	spin_unlock_irqrestore(&phba->hbalock, flags);

	/* Cancel all the IOCBs from the completions list */
	lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT,
			      IOERR_SLI_DOWN);

	spin_lock_irqsave(&phba->hbalock, flags);
	list_splice_init(&phba->elsbuf, &completions);
	phba->elsbuf_cnt = 0;
	phba->elsbuf_prev_cnt = 0;
	spin_unlock_irqrestore(&phba->hbalock, flags);

	while (!list_empty(&completions)) {
		list_remove_head(&completions, buf_ptr,
			struct lpfc_dmabuf, list);
		lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
		kfree(buf_ptr);
	}

	/* Enable softirqs again, done with phba->hbalock */
	local_bh_enable();

	/* Return any active mbox cmds */
	del_timer_sync(&psli->mbox_tmo);

	spin_lock_irqsave(&phba->pport->work_port_lock, flags);
	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
	spin_unlock_irqrestore(&phba->pport->work_port_lock, flags);

	return 1;
}

/**
 * lpfc_sli_pcimem_bcopy - SLI memory copy function
 * @srcp: Source memory pointer.
 * @destp: Destination memory pointer.
 * @cnt: Number of words required to be copied.
 *
 * This function is used for copying data between driver memory
 * and the SLI memory. This function also changes the endianness
 * of each word if native endianness is different from SLI
 * endianness. This function can be called with or without
 * lock.
 **/
void
lpfc_sli_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
{
	uint32_t *src = srcp;
	uint32_t *dest = destp;
	uint32_t ldata;
	int i;

	for (i = 0; i < (int)cnt; i += sizeof (uint32_t)) {
		ldata = *src;
		ldata = le32_to_cpu(ldata);
		*dest = ldata;
		src++;
		dest++;
	}
}


/**
 * lpfc_sli_bemem_bcopy - SLI memory copy function
 * @srcp: Source memory pointer.
 * @destp: Destination memory pointer.
 * @cnt: Number of words required to be copied.
 *
 * This function is used for copying data between a data structure
 * with big endian representation to local endianness.
 * This function can be called with or without lock.
 **/
void
lpfc_sli_bemem_bcopy(void *srcp, void *destp, uint32_t cnt)
{
	uint32_t *src = srcp;
	uint32_t *dest = destp;
	uint32_t ldata;
	int i;

	for (i = 0; i < (int)cnt; i += sizeof(uint32_t)) {
		ldata = *src;
		ldata = be32_to_cpu(ldata);
		*dest = ldata;
		src++;
		dest++;
	}
}

/**
 * lpfc_sli_ringpostbuf_put - Function to add a buffer to postbufq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mp: Pointer to driver buffer object.
 *
 * This function is called with no lock held.
 * It always return zero after adding the buffer to the postbufq
 * buffer list.
 **/
int
lpfc_sli_ringpostbuf_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			 struct lpfc_dmabuf *mp)
{
	/* Stick struct lpfc_dmabuf at end of postbufq so driver can look it up
	   later */
	spin_lock_irq(&phba->hbalock);
	list_add_tail(&mp->list, &pring->postbufq);
	pring->postbufq_cnt++;
	spin_unlock_irq(&phba->hbalock);
	return 0;
}

/**
 * lpfc_sli_get_buffer_tag - allocates a tag for a CMD_QUE_XRI64_CX buffer
 * @phba: Pointer to HBA context object.
 *
 * When HBQ is enabled, buffers are searched based on tags. This function
 * allocates a tag for buffer posted using CMD_QUE_XRI64_CX iocb. The
 * tag is bit wise or-ed with QUE_BUFTAG_BIT to make sure that the tag
 * does not conflict with tags of buffer posted for unsolicited events.
 * The function returns the allocated tag. The function is called with
 * no locks held.
 **/
uint32_t
lpfc_sli_get_buffer_tag(struct lpfc_hba *phba)
{
	spin_lock_irq(&phba->hbalock);
	phba->buffer_tag_count++;
	/*
	 * Always set the QUE_BUFTAG_BIT to distiguish between
	 * a tag assigned by HBQ.
	 */
	phba->buffer_tag_count |= QUE_BUFTAG_BIT;
	spin_unlock_irq(&phba->hbalock);
	return phba->buffer_tag_count;
}

/**
 * lpfc_sli_ring_taggedbuf_get - find HBQ buffer associated with given tag
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @tag: Buffer tag.
 *
 * Buffers posted using CMD_QUE_XRI64_CX iocb are in pring->postbufq
 * list. After HBA DMA data to these buffers, CMD_IOCB_RET_XRI64_CX
 * iocb is posted to the response ring with the tag of the buffer.
 * This function searches the pring->postbufq list using the tag
 * to find buffer associated with CMD_IOCB_RET_XRI64_CX
 * iocb. If the buffer is found then lpfc_dmabuf object of the
 * buffer is returned to the caller else NULL is returned.
 * This function is called with no lock held.
 **/
struct lpfc_dmabuf *
lpfc_sli_ring_taggedbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			uint32_t tag)
{
	struct lpfc_dmabuf *mp, *next_mp;
	struct list_head *slp = &pring->postbufq;

	/* Search postbufq, from the beginning, looking for a match on tag */
	spin_lock_irq(&phba->hbalock);
	list_for_each_entry_safe(mp, next_mp, &pring->postbufq, list) {
		if (mp->buffer_tag == tag) {
			list_del_init(&mp->list);
			pring->postbufq_cnt--;
			spin_unlock_irq(&phba->hbalock);
			return mp;
		}
	}

	spin_unlock_irq(&phba->hbalock);
	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
			"0402 Cannot find virtual addr for buffer tag on "
			"ring %d Data x%lx x%p x%p x%x\n",
			pring->ringno, (unsigned long) tag,
			slp->next, slp->prev, pring->postbufq_cnt);

	return NULL;
}

/**
 * lpfc_sli_ringpostbuf_get - search buffers for unsolicited CT and ELS events
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @phys: DMA address of the buffer.
 *
 * This function searches the buffer list using the dma_address
 * of unsolicited event to find the driver's lpfc_dmabuf object
 * corresponding to the dma_address. The function returns the
 * lpfc_dmabuf object if a buffer is found else it returns NULL.
 * This function is called by the ct and els unsolicited event
 * handlers to get the buffer associated with the unsolicited
 * event.
 *
 * This function is called with no lock held.
 **/
struct lpfc_dmabuf *
lpfc_sli_ringpostbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			 dma_addr_t phys)
{
	struct lpfc_dmabuf *mp, *next_mp;
	struct list_head *slp = &pring->postbufq;

	/* Search postbufq, from the beginning, looking for a match on phys */
	spin_lock_irq(&phba->hbalock);
	list_for_each_entry_safe(mp, next_mp, &pring->postbufq, list) {
		if (mp->phys == phys) {
			list_del_init(&mp->list);
			pring->postbufq_cnt--;
			spin_unlock_irq(&phba->hbalock);
			return mp;
		}
	}

	spin_unlock_irq(&phba->hbalock);
	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
			"0410 Cannot find virtual addr for mapped buf on "
			"ring %d Data x%llx x%p x%p x%x\n",
			pring->ringno, (unsigned long long)phys,
			slp->next, slp->prev, pring->postbufq_cnt);
	return NULL;
}

/**
 * lpfc_sli_abort_els_cmpl - Completion handler for the els abort iocbs
 * @phba: Pointer to HBA context object.
 * @cmdiocb: Pointer to driver command iocb object.
 * @rspiocb: Pointer to driver response iocb object.
 *
 * This function is the completion handler for the abort iocbs for
 * ELS commands. This function is called from the ELS ring event
 * handler with no lock held. This function frees memory resources
 * associated with the abort iocb.
 **/
static void
lpfc_sli_abort_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
			struct lpfc_iocbq *rspiocb)
{
	IOCB_t *irsp = &rspiocb->iocb;
	uint16_t abort_iotag, abort_context;
	struct lpfc_iocbq *abort_iocb = NULL;

	if (irsp->ulpStatus) {

		/*
		 * Assume that the port already completed and returned, or
		 * will return the iocb. Just Log the message.
		 */
		abort_context = cmdiocb->iocb.un.acxri.abortContextTag;
		abort_iotag = cmdiocb->iocb.un.acxri.abortIoTag;

		spin_lock_irq(&phba->hbalock);
		if (phba->sli_rev < LPFC_SLI_REV4) {
			if (irsp->ulpCommand == CMD_ABORT_XRI_CX &&
			    irsp->ulpStatus == IOSTAT_LOCAL_REJECT &&
			    irsp->un.ulpWord[4] == IOERR_ABORT_REQUESTED) {
				spin_unlock_irq(&phba->hbalock);
				goto release_iocb;
			}
			if (abort_iotag != 0 &&
				abort_iotag <= phba->sli.last_iotag)
				abort_iocb =
					phba->sli.iocbq_lookup[abort_iotag];
		} else
			/* For sli4 the abort_tag is the XRI,
			 * so the abort routine puts the iotag  of the iocb
			 * being aborted in the context field of the abort
			 * IOCB.
			 */
			abort_iocb = phba->sli.iocbq_lookup[abort_context];

		lpfc_printf_log(phba, KERN_WARNING, LOG_ELS | LOG_SLI,
				"0327 Cannot abort els iocb %p "
				"with tag %x context %x, abort status %x, "
				"abort code %x\n",
				abort_iocb, abort_iotag, abort_context,
				irsp->ulpStatus, irsp->un.ulpWord[4]);

		spin_unlock_irq(&phba->hbalock);
	}
release_iocb:
	lpfc_sli_release_iocbq(phba, cmdiocb);
	return;
}

/**
 * lpfc_ignore_els_cmpl - Completion handler for aborted ELS command
 * @phba: Pointer to HBA context object.
 * @cmdiocb: Pointer to driver command iocb object.
 * @rspiocb: Pointer to driver response iocb object.
 *
 * The function is called from SLI ring event handler with no
 * lock held. This function is the completion handler for ELS commands
 * which are aborted. The function frees memory resources used for
 * the aborted ELS commands.
 **/
static void
lpfc_ignore_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
		     struct lpfc_iocbq *rspiocb)
{
	IOCB_t *irsp = &rspiocb->iocb;

	/* ELS cmd tag <ulpIoTag> completes */
	lpfc_printf_log(phba, KERN_INFO, LOG_ELS,
			"0139 Ignoring ELS cmd tag x%x completion Data: "
			"x%x x%x x%x\n",
			irsp->ulpIoTag, irsp->ulpStatus,
			irsp->un.ulpWord[4], irsp->ulpTimeout);
	if (cmdiocb->iocb.ulpCommand == CMD_GEN_REQUEST64_CR)
		lpfc_ct_free_iocb(phba, cmdiocb);
	else
		lpfc_els_free_iocb(phba, cmdiocb);
	return;
}

/**
 * lpfc_sli_abort_iotag_issue - Issue abort for a command iocb
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @cmdiocb: Pointer to driver command iocb object.
 *
 * This function issues an abort iocb for the provided command iocb down to
 * the port. Other than the case the outstanding command iocb is an abort
 * request, this function issues abort out unconditionally. This function is
 * called with hbalock held. The function returns 0 when it fails due to
 * memory allocation failure or when the command iocb is an abort request.
 **/
static int
lpfc_sli_abort_iotag_issue(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			   struct lpfc_iocbq *cmdiocb)
{
	struct lpfc_vport *vport = cmdiocb->vport;
	struct lpfc_iocbq *abtsiocbp;
	IOCB_t *icmd = NULL;
	IOCB_t *iabt = NULL;
	int retval;
	unsigned long iflags;
	struct lpfc_nodelist *ndlp;

	lockdep_assert_held(&phba->hbalock);

	/*
	 * There are certain command types we don't want to abort.  And we
	 * don't want to abort commands that are already in the process of
	 * being aborted.
	 */
	icmd = &cmdiocb->iocb;
	if (icmd->ulpCommand == CMD_ABORT_XRI_CN ||
	    icmd->ulpCommand == CMD_CLOSE_XRI_CN ||
	    (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0)
		return 0;

	/* issue ABTS for this IOCB based on iotag */
	abtsiocbp = __lpfc_sli_get_iocbq(phba);
	if (abtsiocbp == NULL)
		return 0;

	/* This signals the response to set the correct status
	 * before calling the completion handler
	 */
	cmdiocb->iocb_flag |= LPFC_DRIVER_ABORTED;

	iabt = &abtsiocbp->iocb;
	iabt->un.acxri.abortType = ABORT_TYPE_ABTS;
	iabt->un.acxri.abortContextTag = icmd->ulpContext;
	if (phba->sli_rev == LPFC_SLI_REV4) {
		iabt->un.acxri.abortIoTag = cmdiocb->sli4_xritag;
		iabt->un.acxri.abortContextTag = cmdiocb->iotag;
	} else {
		iabt->un.acxri.abortIoTag = icmd->ulpIoTag;
		if (pring->ringno == LPFC_ELS_RING) {
			ndlp = (struct lpfc_nodelist *)(cmdiocb->context1);
			iabt->un.acxri.abortContextTag = ndlp->nlp_rpi;
		}
	}
	iabt->ulpLe = 1;
	iabt->ulpClass = icmd->ulpClass;

	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
	abtsiocbp->hba_wqidx = cmdiocb->hba_wqidx;
	if (cmdiocb->iocb_flag & LPFC_IO_FCP)
		abtsiocbp->iocb_flag |= LPFC_USE_FCPWQIDX;
	if (cmdiocb->iocb_flag & LPFC_IO_FOF)
		abtsiocbp->iocb_flag |= LPFC_IO_FOF;

	if (phba->link_state >= LPFC_LINK_UP)
		iabt->ulpCommand = CMD_ABORT_XRI_CN;
	else
		iabt->ulpCommand = CMD_CLOSE_XRI_CN;

	abtsiocbp->iocb_cmpl = lpfc_sli_abort_els_cmpl;
	abtsiocbp->vport = vport;

	lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI,
			 "0339 Abort xri x%x, original iotag x%x, "
			 "abort cmd iotag x%x\n",
			 iabt->un.acxri.abortIoTag,
			 iabt->un.acxri.abortContextTag,
			 abtsiocbp->iotag);

	if (phba->sli_rev == LPFC_SLI_REV4) {
		pring = lpfc_sli4_calc_ring(phba, abtsiocbp);
		if (unlikely(pring == NULL))
			return 0;
		/* Note: both hbalock and ring_lock need to be set here */
		spin_lock_irqsave(&pring->ring_lock, iflags);
		retval = __lpfc_sli_issue_iocb(phba, pring->ringno,
			abtsiocbp, 0);
		spin_unlock_irqrestore(&pring->ring_lock, iflags);
	} else {
		retval = __lpfc_sli_issue_iocb(phba, pring->ringno,
			abtsiocbp, 0);
	}

	if (retval)
		__lpfc_sli_release_iocbq(phba, abtsiocbp);

	/*
	 * Caller to this routine should check for IOCB_ERROR
	 * and handle it properly.  This routine no longer removes
	 * iocb off txcmplq and call compl in case of IOCB_ERROR.
	 */
	return retval;
}

/**
 * lpfc_sli_issue_abort_iotag - Abort function for a command iocb
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @cmdiocb: Pointer to driver command iocb object.
 *
 * This function issues an abort iocb for the provided command iocb. In case
 * of unloading, the abort iocb will not be issued to commands on the ELS
 * ring. Instead, the callback function shall be changed to those commands
 * so that nothing happens when them finishes. This function is called with
 * hbalock held. The function returns 0 when the command iocb is an abort
 * request.
 **/
int
lpfc_sli_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
			   struct lpfc_iocbq *cmdiocb)
{
	struct lpfc_vport *vport = cmdiocb->vport;
	int retval = IOCB_ERROR;
	IOCB_t *icmd = NULL;

	lockdep_assert_held(&phba->hbalock);

	/*
	 * There are certain command types we don't want to abort.  And we
	 * don't want to abort commands that are already in the process of
	 * being aborted.
	 */
	icmd = &cmdiocb->iocb;
	if (icmd->ulpCommand == CMD_ABORT_XRI_CN ||
	    icmd->ulpCommand == CMD_CLOSE_XRI_CN ||
	    (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0)
		return 0;

	if (!pring) {
		if (cmdiocb->iocb_flag & LPFC_IO_FABRIC)
			cmdiocb->fabric_iocb_cmpl = lpfc_ignore_els_cmpl;
		else
			cmdiocb->iocb_cmpl = lpfc_ignore_els_cmpl;
		goto abort_iotag_exit;
	}

	/*
	 * If we're unloading, don't abort iocb on the ELS ring, but change
	 * the callback so that nothing happens when it finishes.
	 */
	if ((vport->load_flag & FC_UNLOADING) &&
	    (pring->ringno == LPFC_ELS_RING)) {
		if (cmdiocb->iocb_flag & LPFC_IO_FABRIC)
			cmdiocb->fabric_iocb_cmpl = lpfc_ignore_els_cmpl;
		else
			cmdiocb->iocb_cmpl = lpfc_ignore_els_cmpl;
		goto abort_iotag_exit;
	}

	/* Now, we try to issue the abort to the cmdiocb out */
	retval = lpfc_sli_abort_iotag_issue(phba, pring, cmdiocb);

abort_iotag_exit:
	/*
	 * Caller to this routine should check for IOCB_ERROR
	 * and handle it properly.  This routine no longer removes
	 * iocb off txcmplq and call compl in case of IOCB_ERROR.
	 */
	return retval;
}

/**
 * lpfc_sli_hba_iocb_abort - Abort all iocbs to an hba.
 * @phba: pointer to lpfc HBA data structure.
 *
 * This routine will abort all pending and outstanding iocbs to an HBA.
 **/
void
lpfc_sli_hba_iocb_abort(struct lpfc_hba *phba)
{
	struct lpfc_sli *psli = &phba->sli;
	struct lpfc_sli_ring *pring;
	struct lpfc_queue *qp = NULL;
	int i;

	if (phba->sli_rev != LPFC_SLI_REV4) {
		for (i = 0; i < psli->num_rings; i++) {
			pring = &psli->sli3_ring[i];
			lpfc_sli_abort_iocb_ring(phba, pring);
		}
		return;
	}
	list_for_each_entry(qp, &phba->sli4_hba.lpfc_wq_list, wq_list) {
		pring = qp->pring;
		if (!pring)
			continue;
		lpfc_sli_abort_iocb_ring(phba, pring);
	}
}

/**
 * lpfc_sli_validate_fcp_iocb - find commands associated with a vport or LUN
 * @iocbq: Pointer to driver iocb object.
 * @vport: Pointer to driver virtual port object.
 * @tgt_id: SCSI ID of the target.
 * @lun_id: LUN ID of the scsi device.
 * @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST
 *
 * This function acts as an iocb filter for functions which abort or count
 * all FCP iocbs pending on a lun/SCSI target/SCSI host. It will return
 * 0 if the filtering criteria is met for the given iocb and will return
 * 1 if the filtering criteria is not met.
 * If ctx_cmd == LPFC_CTX_LUN, the function returns 0 only if the
 * given iocb is for the SCSI device specified by vport, tgt_id and
 * lun_id parameter.
 * If ctx_cmd == LPFC_CTX_TGT,  the function returns 0 only if the
 * given iocb is for the SCSI target specified by vport and tgt_id
 * parameters.
 * If ctx_cmd == LPFC_CTX_HOST, the function returns 0 only if the
 * given iocb is for the SCSI host associated with the given vport.
 * This function is called with no locks held.
 **/
static int
lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport,
			   uint16_t tgt_id, uint64_t lun_id,
			   lpfc_ctx_cmd ctx_cmd)
{
	struct lpfc_io_buf *lpfc_cmd;
	int rc = 1;

	if (iocbq->vport != vport)
		return rc;

	if (!(iocbq->iocb_flag &  LPFC_IO_FCP) ||
	    !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ))
		return rc;

	lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);

	if (lpfc_cmd->pCmd == NULL)
		return rc;

	switch (ctx_cmd) {
	case LPFC_CTX_LUN:
		if ((lpfc_cmd->rdata) && (lpfc_cmd->rdata->pnode) &&
		    (lpfc_cmd->rdata->pnode->nlp_sid == tgt_id) &&
		    (scsilun_to_int(&lpfc_cmd->fcp_cmnd->fcp_lun) == lun_id))
			rc = 0;
		break;
	case LPFC_CTX_TGT:
		if ((lpfc_cmd->rdata) && (lpfc_cmd->rdata->pnode) &&
		    (lpfc_cmd->rdata->pnode->nlp_sid == tgt_id))
			rc = 0;
		break;
	case LPFC_CTX_HOST:
		rc = 0;
		break;
	default:
		printk(KERN_ERR "%s: Unknown context cmd type, value %d\n",
			__func__, ctx_cmd);
		break;
	}

	return rc;
}

/**
 * lpfc_sli_sum_iocb - Function to count the number of FCP iocbs pending
 * @vport: Pointer to virtual port.
 * @tgt_id: SCSI ID of the target.
 * @lun_id: LUN ID of the scsi device.
 * @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
 *
 * This function returns number of FCP commands pending for the vport.
 * When ctx_cmd == LPFC_CTX_LUN, the function returns number of FCP
 * commands pending on the vport associated with SCSI device specified
 * by tgt_id and lun_id parameters.
 * When ctx_cmd == LPFC_CTX_TGT, the function returns number of FCP
 * commands pending on the vport associated with SCSI target specified
 * by tgt_id parameter.
 * When ctx_cmd == LPFC_CTX_HOST, the function returns number of FCP
 * commands pending on the vport.
 * This function returns the number of iocbs which satisfy the filter.
 * This function is called without any lock held.
 **/
int
lpfc_sli_sum_iocb(struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id,
		  lpfc_ctx_cmd ctx_cmd)
{
	struct lpfc_hba *phba = vport->phba;
	struct lpfc_iocbq *iocbq;
	int sum, i;

	spin_lock_irq(&phba->hbalock);
	for (i = 1, sum = 0; i <= phba->sli.last_iotag; i++) {
		iocbq = phba->sli.iocbq_lookup[i];

		if (lpfc_sli_validate_fcp_iocb (iocbq, vport, tgt_id, lun_id,
						ctx_cmd) == 0)
			sum++;
	}
	spin_unlock_irq(&phba->hbalock);

	return sum;
}

/**
 * lpfc_sli_abort_fcp_cmpl - Completion handler function for aborted FCP IOCBs
 * @phba: Pointer to HBA context object
 * @cmdiocb: Pointer to command iocb object.
 * @rspiocb: Pointer to response iocb object.
 *
 * This function is called when an aborted FCP iocb completes. This
 * function is called by the ring event handler with no lock held.
 * This function frees the iocb.
 **/
void
lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
			struct lpfc_iocbq *rspiocb)
{
	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"3096 ABORT_XRI_CN completing on rpi x%x "
			"original iotag x%x, abort cmd iotag x%x "
			"status 0x%x, reason 0x%x\n",
			cmdiocb->iocb.un.acxri.abortContextTag,
			cmdiocb->iocb.un.acxri.abortIoTag,
			cmdiocb->iotag, rspiocb->iocb.ulpStatus,
			rspiocb->iocb.un.ulpWord[4]);
	lpfc_sli_release_iocbq(phba, cmdiocb);
	return;
}

/**
 * lpfc_sli_abort_iocb - issue abort for all commands on a host/target/LUN
 * @vport: Pointer to virtual port.
 * @pring: Pointer to driver SLI ring object.
 * @tgt_id: SCSI ID of the target.
 * @lun_id: LUN ID of the scsi device.
 * @abort_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
 *
 * This function sends an abort command for every SCSI command
 * associated with the given virtual port pending on the ring
 * filtered by lpfc_sli_validate_fcp_iocb function.
 * When abort_cmd == LPFC_CTX_LUN, the function sends abort only to the
 * FCP iocbs associated with lun specified by tgt_id and lun_id
 * parameters
 * When abort_cmd == LPFC_CTX_TGT, the function sends abort only to the
 * FCP iocbs associated with SCSI target specified by tgt_id parameter.
 * When abort_cmd == LPFC_CTX_HOST, the function sends abort to all
 * FCP iocbs associated with virtual port.
 * This function returns number of iocbs it failed to abort.
 * This function is called with no locks held.
 **/
int
lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
		    uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd abort_cmd)
{
	struct lpfc_hba *phba = vport->phba;
	struct lpfc_iocbq *iocbq;
	struct lpfc_iocbq *abtsiocb;
	struct lpfc_sli_ring *pring_s4;
	IOCB_t *cmd = NULL;
	int errcnt = 0, ret_val = 0;
	int i;

	/* all I/Os are in process of being flushed */
	if (phba->hba_flag & HBA_FCP_IOQ_FLUSH)
		return errcnt;

	for (i = 1; i <= phba->sli.last_iotag; i++) {
		iocbq = phba->sli.iocbq_lookup[i];

		if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
					       abort_cmd) != 0)
			continue;

		/*
		 * If the iocbq is already being aborted, don't take a second
		 * action, but do count it.
		 */
		if (iocbq->iocb_flag & LPFC_DRIVER_ABORTED)
			continue;

		/* issue ABTS for this IOCB based on iotag */
		abtsiocb = lpfc_sli_get_iocbq(phba);
		if (abtsiocb == NULL) {
			errcnt++;
			continue;
		}

		/* indicate the IO is being aborted by the driver. */
		iocbq->iocb_flag |= LPFC_DRIVER_ABORTED;

		cmd = &iocbq->iocb;
		abtsiocb->iocb.un.acxri.abortType = ABORT_TYPE_ABTS;
		abtsiocb->iocb.un.acxri.abortContextTag = cmd->ulpContext;
		if (phba->sli_rev == LPFC_SLI_REV4)
			abtsiocb->iocb.un.acxri.abortIoTag = iocbq->sli4_xritag;
		else
			abtsiocb->iocb.un.acxri.abortIoTag = cmd->ulpIoTag;
		abtsiocb->iocb.ulpLe = 1;
		abtsiocb->iocb.ulpClass = cmd->ulpClass;
		abtsiocb->vport = vport;

		/* ABTS WQE must go to the same WQ as the WQE to be aborted */
		abtsiocb->hba_wqidx = iocbq->hba_wqidx;
		if (iocbq->iocb_flag & LPFC_IO_FCP)
			abtsiocb->iocb_flag |= LPFC_USE_FCPWQIDX;
		if (iocbq->iocb_flag & LPFC_IO_FOF)
			abtsiocb->iocb_flag |= LPFC_IO_FOF;

		if (lpfc_is_link_up(phba))
			abtsiocb->iocb.ulpCommand = CMD_ABORT_XRI_CN;
		else
			abtsiocb->iocb.ulpCommand = CMD_CLOSE_XRI_CN;

		/* Setup callback routine and issue the command. */
		abtsiocb->iocb_cmpl = lpfc_sli_abort_fcp_cmpl;
		if (phba->sli_rev == LPFC_SLI_REV4) {
			pring_s4 = lpfc_sli4_calc_ring(phba, iocbq);
			if (!pring_s4)
				continue;
			ret_val = lpfc_sli_issue_iocb(phba, pring_s4->ringno,
						      abtsiocb, 0);
		} else
			ret_val = lpfc_sli_issue_iocb(phba, pring->ringno,
						      abtsiocb, 0);
		if (ret_val == IOCB_ERROR) {
			lpfc_sli_release_iocbq(phba, abtsiocb);
			errcnt++;
			continue;
		}
	}

	return errcnt;
}

/**
 * lpfc_sli_abort_taskmgmt - issue abort for all commands on a host/target/LUN
 * @vport: Pointer to virtual port.
 * @pring: Pointer to driver SLI ring object.
 * @tgt_id: SCSI ID of the target.
 * @lun_id: LUN ID of the scsi device.
 * @taskmgmt_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST.
 *
 * This function sends an abort command for every SCSI command
 * associated with the given virtual port pending on the ring
 * filtered by lpfc_sli_validate_fcp_iocb function.
 * When taskmgmt_cmd == LPFC_CTX_LUN, the function sends abort only to the
 * FCP iocbs associated with lun specified by tgt_id and lun_id
 * parameters
 * When taskmgmt_cmd == LPFC_CTX_TGT, the function sends abort only to the
 * FCP iocbs associated with SCSI target specified by tgt_id parameter.
 * When taskmgmt_cmd == LPFC_CTX_HOST, the function sends abort to all
 * FCP iocbs associated with virtual port.
 * This function returns number of iocbs it aborted .
 * This function is called with no locks held right after a taskmgmt
 * command is sent.
 **/
int
lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
			uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd cmd)
{
	struct lpfc_hba *phba = vport->phba;
	struct lpfc_io_buf *lpfc_cmd;
	struct lpfc_iocbq *abtsiocbq;
	struct lpfc_nodelist *ndlp;
	struct lpfc_iocbq *iocbq;
	IOCB_t *icmd;
	int sum, i, ret_val;
	unsigned long iflags;
	struct lpfc_sli_ring *pring_s4 = NULL;

	spin_lock_irqsave(&phba->hbalock, iflags);

	/* all I/Os are in process of being flushed */
	if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) {
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		return 0;
	}
	sum = 0;

	for (i = 1; i <= phba->sli.last_iotag; i++) {
		iocbq = phba->sli.iocbq_lookup[i];

		if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id,
					       cmd) != 0)
			continue;

		/* Guard against IO completion being called at same time */
		lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
		spin_lock(&lpfc_cmd->buf_lock);

		if (!lpfc_cmd->pCmd) {
			spin_unlock(&lpfc_cmd->buf_lock);
			continue;
		}

		if (phba->sli_rev == LPFC_SLI_REV4) {
			pring_s4 =
			    phba->sli4_hba.hdwq[iocbq->hba_wqidx].fcp_wq->pring;
			if (!pring_s4) {
				spin_unlock(&lpfc_cmd->buf_lock);
				continue;
			}
			/* Note: both hbalock and ring_lock must be set here */
			spin_lock(&pring_s4->ring_lock);
		}

		/*
		 * If the iocbq is already being aborted, don't take a second
		 * action, but do count it.
		 */
		if ((iocbq->iocb_flag & LPFC_DRIVER_ABORTED) ||
		    !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
			if (phba->sli_rev == LPFC_SLI_REV4)
				spin_unlock(&pring_s4->ring_lock);
			spin_unlock(&lpfc_cmd->buf_lock);
			continue;
		}

		/* issue ABTS for this IOCB based on iotag */
		abtsiocbq = __lpfc_sli_get_iocbq(phba);
		if (!abtsiocbq) {
			if (phba->sli_rev == LPFC_SLI_REV4)
				spin_unlock(&pring_s4->ring_lock);
			spin_unlock(&lpfc_cmd->buf_lock);
			continue;
		}

		icmd = &iocbq->iocb;
		abtsiocbq->iocb.un.acxri.abortType = ABORT_TYPE_ABTS;
		abtsiocbq->iocb.un.acxri.abortContextTag = icmd->ulpContext;
		if (phba->sli_rev == LPFC_SLI_REV4)
			abtsiocbq->iocb.un.acxri.abortIoTag =
							 iocbq->sli4_xritag;
		else
			abtsiocbq->iocb.un.acxri.abortIoTag = icmd->ulpIoTag;
		abtsiocbq->iocb.ulpLe = 1;
		abtsiocbq->iocb.ulpClass = icmd->ulpClass;
		abtsiocbq->vport = vport;

		/* ABTS WQE must go to the same WQ as the WQE to be aborted */
		abtsiocbq->hba_wqidx = iocbq->hba_wqidx;
		if (iocbq->iocb_flag & LPFC_IO_FCP)
			abtsiocbq->iocb_flag |= LPFC_USE_FCPWQIDX;
		if (iocbq->iocb_flag & LPFC_IO_FOF)
			abtsiocbq->iocb_flag |= LPFC_IO_FOF;

		ndlp = lpfc_cmd->rdata->pnode;

		if (lpfc_is_link_up(phba) &&
		    (ndlp && ndlp->nlp_state == NLP_STE_MAPPED_NODE))
			abtsiocbq->iocb.ulpCommand = CMD_ABORT_XRI_CN;
		else
			abtsiocbq->iocb.ulpCommand = CMD_CLOSE_XRI_CN;

		/* Setup callback routine and issue the command. */
		abtsiocbq->iocb_cmpl = lpfc_sli_abort_fcp_cmpl;

		/*
		 * Indicate the IO is being aborted by the driver and set
		 * the caller's flag into the aborted IO.
		 */
		iocbq->iocb_flag |= LPFC_DRIVER_ABORTED;

		if (phba->sli_rev == LPFC_SLI_REV4) {
			ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
							abtsiocbq, 0);
			spin_unlock(&pring_s4->ring_lock);
		} else {
			ret_val = __lpfc_sli_issue_iocb(phba, pring->ringno,
							abtsiocbq, 0);
		}

		spin_unlock(&lpfc_cmd->buf_lock);

		if (ret_val == IOCB_ERROR)
			__lpfc_sli_release_iocbq(phba, abtsiocbq);
		else
			sum++;
	}
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	return sum;
}

/**
 * lpfc_sli_wake_iocb_wait - lpfc_sli_issue_iocb_wait's completion handler
 * @phba: Pointer to HBA context object.
 * @cmdiocbq: Pointer to command iocb.
 * @rspiocbq: Pointer to response iocb.
 *
 * This function is the completion handler for iocbs issued using
 * lpfc_sli_issue_iocb_wait function. This function is called by the
 * ring event handler function without any lock held. This function
 * can be called from both worker thread context and interrupt
 * context. This function also can be called from other thread which
 * cleans up the SLI layer objects.
 * This function copy the contents of the response iocb to the
 * response iocb memory object provided by the caller of
 * lpfc_sli_issue_iocb_wait and then wakes up the thread which
 * sleeps for the iocb completion.
 **/
static void
lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba,
			struct lpfc_iocbq *cmdiocbq,
			struct lpfc_iocbq *rspiocbq)
{
	wait_queue_head_t *pdone_q;
	unsigned long iflags;
	struct lpfc_io_buf *lpfc_cmd;

	spin_lock_irqsave(&phba->hbalock, iflags);
	if (cmdiocbq->iocb_flag & LPFC_IO_WAKE_TMO) {

		/*
		 * A time out has occurred for the iocb.  If a time out
		 * completion handler has been supplied, call it.  Otherwise,
		 * just free the iocbq.
		 */

		spin_unlock_irqrestore(&phba->hbalock, iflags);
		cmdiocbq->iocb_cmpl = cmdiocbq->wait_iocb_cmpl;
		cmdiocbq->wait_iocb_cmpl = NULL;
		if (cmdiocbq->iocb_cmpl)
			(cmdiocbq->iocb_cmpl)(phba, cmdiocbq, NULL);
		else
			lpfc_sli_release_iocbq(phba, cmdiocbq);
		return;
	}

	cmdiocbq->iocb_flag |= LPFC_IO_WAKE;
	if (cmdiocbq->context2 && rspiocbq)
		memcpy(&((struct lpfc_iocbq *)cmdiocbq->context2)->iocb,
		       &rspiocbq->iocb, sizeof(IOCB_t));

	/* Set the exchange busy flag for task management commands */
	if ((cmdiocbq->iocb_flag & LPFC_IO_FCP) &&
		!(cmdiocbq->iocb_flag & LPFC_IO_LIBDFC)) {
		lpfc_cmd = container_of(cmdiocbq, struct lpfc_io_buf,
			cur_iocbq);
		lpfc_cmd->exch_busy = rspiocbq->iocb_flag & LPFC_EXCHANGE_BUSY;
	}

	pdone_q = cmdiocbq->context_un.wait_queue;
	if (pdone_q)
		wake_up(pdone_q);
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	return;
}

/**
 * lpfc_chk_iocb_flg - Test IOCB flag with lock held.
 * @phba: Pointer to HBA context object..
 * @piocbq: Pointer to command iocb.
 * @flag: Flag to test.
 *
 * This routine grabs the hbalock and then test the iocb_flag to
 * see if the passed in flag is set.
 * Returns:
 * 1 if flag is set.
 * 0 if flag is not set.
 **/
static int
lpfc_chk_iocb_flg(struct lpfc_hba *phba,
		 struct lpfc_iocbq *piocbq, uint32_t flag)
{
	unsigned long iflags;
	int ret;

	spin_lock_irqsave(&phba->hbalock, iflags);
	ret = piocbq->iocb_flag & flag;
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	return ret;

}

/**
 * lpfc_sli_issue_iocb_wait - Synchronous function to issue iocb commands
 * @phba: Pointer to HBA context object..
 * @pring: Pointer to sli ring.
 * @piocb: Pointer to command iocb.
 * @prspiocbq: Pointer to response iocb.
 * @timeout: Timeout in number of seconds.
 *
 * This function issues the iocb to firmware and waits for the
 * iocb to complete. The iocb_cmpl field of the shall be used
 * to handle iocbs which time out. If the field is NULL, the
 * function shall free the iocbq structure.  If more clean up is
 * needed, the caller is expected to provide a completion function
 * that will provide the needed clean up.  If the iocb command is
 * not completed within timeout seconds, the function will either
 * free the iocbq structure (if iocb_cmpl == NULL) or execute the
 * completion function set in the iocb_cmpl field and then return
 * a status of IOCB_TIMEDOUT.  The caller should not free the iocb
 * resources if this function returns IOCB_TIMEDOUT.
 * The function waits for the iocb completion using an
 * non-interruptible wait.
 * This function will sleep while waiting for iocb completion.
 * So, this function should not be called from any context which
 * does not allow sleeping. Due to the same reason, this function
 * cannot be called with interrupt disabled.
 * This function assumes that the iocb completions occur while
 * this function sleep. So, this function cannot be called from
 * the thread which process iocb completion for this ring.
 * This function clears the iocb_flag of the iocb object before
 * issuing the iocb and the iocb completion handler sets this
 * flag and wakes this thread when the iocb completes.
 * The contents of the response iocb will be copied to prspiocbq
 * by the completion handler when the command completes.
 * This function returns IOCB_SUCCESS when success.
 * This function is called with no lock held.
 **/
int
lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba,
			 uint32_t ring_number,
			 struct lpfc_iocbq *piocb,
			 struct lpfc_iocbq *prspiocbq,
			 uint32_t timeout)
{
	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_q);
	long timeleft, timeout_req = 0;
	int retval = IOCB_SUCCESS;
	uint32_t creg_val;
	struct lpfc_iocbq *iocb;
	int txq_cnt = 0;
	int txcmplq_cnt = 0;
	struct lpfc_sli_ring *pring;
	unsigned long iflags;
	bool iocb_completed = true;

	if (phba->sli_rev >= LPFC_SLI_REV4)
		pring = lpfc_sli4_calc_ring(phba, piocb);
	else
		pring = &phba->sli.sli3_ring[ring_number];
	/*
	 * If the caller has provided a response iocbq buffer, then context2
	 * is NULL or its an error.
	 */
	if (prspiocbq) {
		if (piocb->context2)
			return IOCB_ERROR;
		piocb->context2 = prspiocbq;
	}

	piocb->wait_iocb_cmpl = piocb->iocb_cmpl;
	piocb->iocb_cmpl = lpfc_sli_wake_iocb_wait;
	piocb->context_un.wait_queue = &done_q;
	piocb->iocb_flag &= ~(LPFC_IO_WAKE | LPFC_IO_WAKE_TMO);

	if (phba->cfg_poll & DISABLE_FCP_RING_INT) {
		if (lpfc_readl(phba->HCregaddr, &creg_val))
			return IOCB_ERROR;
		creg_val |= (HC_R0INT_ENA << LPFC_FCP_RING);
		writel(creg_val, phba->HCregaddr);
		readl(phba->HCregaddr); /* flush */
	}

	retval = lpfc_sli_issue_iocb(phba, ring_number, piocb,
				     SLI_IOCB_RET_IOCB);
	if (retval == IOCB_SUCCESS) {
		timeout_req = msecs_to_jiffies(timeout * 1000);
		timeleft = wait_event_timeout(done_q,
				lpfc_chk_iocb_flg(phba, piocb, LPFC_IO_WAKE),
				timeout_req);
		spin_lock_irqsave(&phba->hbalock, iflags);
		if (!(piocb->iocb_flag & LPFC_IO_WAKE)) {

			/*
			 * IOCB timed out.  Inform the wake iocb wait
			 * completion function and set local status
			 */

			iocb_completed = false;
			piocb->iocb_flag |= LPFC_IO_WAKE_TMO;
		}
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		if (iocb_completed) {
			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
					"0331 IOCB wake signaled\n");
			/* Note: we are not indicating if the IOCB has a success
			 * status or not - that's for the caller to check.
			 * IOCB_SUCCESS means just that the command was sent and
			 * completed. Not that it completed successfully.
			 * */
		} else if (timeleft == 0) {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"0338 IOCB wait timeout error - no "
					"wake response Data x%x\n", timeout);
			retval = IOCB_TIMEDOUT;
		} else {
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"0330 IOCB wake NOT set, "
					"Data x%x x%lx\n",
					timeout, (timeleft / jiffies));
			retval = IOCB_TIMEDOUT;
		}
	} else if (retval == IOCB_BUSY) {
		if (phba->cfg_log_verbose & LOG_SLI) {
			list_for_each_entry(iocb, &pring->txq, list) {
				txq_cnt++;
			}
			list_for_each_entry(iocb, &pring->txcmplq, list) {
				txcmplq_cnt++;
			}
			lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"2818 Max IOCBs %d txq cnt %d txcmplq cnt %d\n",
				phba->iocb_cnt, txq_cnt, txcmplq_cnt);
		}
		return retval;
	} else {
		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"0332 IOCB wait issue failed, Data x%x\n",
				retval);
		retval = IOCB_ERROR;
	}

	if (phba->cfg_poll & DISABLE_FCP_RING_INT) {
		if (lpfc_readl(phba->HCregaddr, &creg_val))
			return IOCB_ERROR;
		creg_val &= ~(HC_R0INT_ENA << LPFC_FCP_RING);
		writel(creg_val, phba->HCregaddr);
		readl(phba->HCregaddr); /* flush */
	}

	if (prspiocbq)
		piocb->context2 = NULL;

	piocb->context_un.wait_queue = NULL;
	piocb->iocb_cmpl = NULL;
	return retval;
}

/**
 * lpfc_sli_issue_mbox_wait - Synchronous function to issue mailbox
 * @phba: Pointer to HBA context object.
 * @pmboxq: Pointer to driver mailbox object.
 * @timeout: Timeout in number of seconds.
 *
 * This function issues the mailbox to firmware and waits for the
 * mailbox command to complete. If the mailbox command is not
 * completed within timeout seconds, it returns MBX_TIMEOUT.
 * The function waits for the mailbox completion using an
 * interruptible wait. If the thread is woken up due to a
 * signal, MBX_TIMEOUT error is returned to the caller. Caller
 * should not free the mailbox resources, if this function returns
 * MBX_TIMEOUT.
 * This function will sleep while waiting for mailbox completion.
 * So, this function should not be called from any context which
 * does not allow sleeping. Due to the same reason, this function
 * cannot be called with interrupt disabled.
 * This function assumes that the mailbox completion occurs while
 * this function sleep. So, this function cannot be called from
 * the worker thread which processes mailbox completion.
 * This function is called in the context of HBA management
 * applications.
 * This function returns MBX_SUCCESS when successful.
 * This function is called with no lock held.
 **/
int
lpfc_sli_issue_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq,
			 uint32_t timeout)
{
	struct completion mbox_done;
	int retval;
	unsigned long flag;

	pmboxq->mbox_flag &= ~LPFC_MBX_WAKE;
	/* setup wake call as IOCB callback */
	pmboxq->mbox_cmpl = lpfc_sli_wake_mbox_wait;

	/* setup context3 field to pass wait_queue pointer to wake function  */
	init_completion(&mbox_done);
	pmboxq->context3 = &mbox_done;
	/* now issue the command */
	retval = lpfc_sli_issue_mbox(phba, pmboxq, MBX_NOWAIT);
	if (retval == MBX_BUSY || retval == MBX_SUCCESS) {
		wait_for_completion_timeout(&mbox_done,
					    msecs_to_jiffies(timeout * 1000));

		spin_lock_irqsave(&phba->hbalock, flag);
		pmboxq->context3 = NULL;
		/*
		 * if LPFC_MBX_WAKE flag is set the mailbox is completed
		 * else do not free the resources.
		 */
		if (pmboxq->mbox_flag & LPFC_MBX_WAKE) {
			retval = MBX_SUCCESS;
		} else {
			retval = MBX_TIMEOUT;
			pmboxq->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
		}
		spin_unlock_irqrestore(&phba->hbalock, flag);
	}
	return retval;
}

/**
 * lpfc_sli_mbox_sys_shutdown - shutdown mailbox command sub-system
 * @phba: Pointer to HBA context.
 *
 * This function is called to shutdown the driver's mailbox sub-system.
 * It first marks the mailbox sub-system is in a block state to prevent
 * the asynchronous mailbox command from issued off the pending mailbox
 * command queue. If the mailbox command sub-system shutdown is due to
 * HBA error conditions such as EEH or ERATT, this routine shall invoke
 * the mailbox sub-system flush routine to forcefully bring down the
 * mailbox sub-system. Otherwise, if it is due to normal condition (such
 * as with offline or HBA function reset), this routine will wait for the
 * outstanding mailbox command to complete before invoking the mailbox
 * sub-system flush routine to gracefully bring down mailbox sub-system.
 **/
void
lpfc_sli_mbox_sys_shutdown(struct lpfc_hba *phba, int mbx_action)
{
	struct lpfc_sli *psli = &phba->sli;
	unsigned long timeout;

	if (mbx_action == LPFC_MBX_NO_WAIT) {
		/* delay 100ms for port state */
		msleep(100);
		lpfc_sli_mbox_sys_flush(phba);
		return;
	}
	timeout = msecs_to_jiffies(LPFC_MBOX_TMO * 1000) + jiffies;

	/* Disable softirqs, including timers from obtaining phba->hbalock */
	local_bh_disable();

	spin_lock_irq(&phba->hbalock);
	psli->sli_flag |= LPFC_SLI_ASYNC_MBX_BLK;

	if (psli->sli_flag & LPFC_SLI_ACTIVE) {
		/* Determine how long we might wait for the active mailbox
		 * command to be gracefully completed by firmware.
		 */
		if (phba->sli.mbox_active)
			timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba,
						phba->sli.mbox_active) *
						1000) + jiffies;
		spin_unlock_irq(&phba->hbalock);

		/* Enable softirqs again, done with phba->hbalock */
		local_bh_enable();

		while (phba->sli.mbox_active) {
			/* Check active mailbox complete status every 2ms */
			msleep(2);
			if (time_after(jiffies, timeout))
				/* Timeout, let the mailbox flush routine to
				 * forcefully release active mailbox command
				 */
				break;
		}
	} else {
		spin_unlock_irq(&phba->hbalock);

		/* Enable softirqs again, done with phba->hbalock */
		local_bh_enable();
	}

	lpfc_sli_mbox_sys_flush(phba);
}

/**
 * lpfc_sli_eratt_read - read sli-3 error attention events
 * @phba: Pointer to HBA context.
 *
 * This function is called to read the SLI3 device error attention registers
 * for possible error attention events. The caller must hold the hostlock
 * with spin_lock_irq().
 *
 * This function returns 1 when there is Error Attention in the Host Attention
 * Register and returns 0 otherwise.
 **/
static int
lpfc_sli_eratt_read(struct lpfc_hba *phba)
{
	uint32_t ha_copy;

	/* Read chip Host Attention (HA) register */
	if (lpfc_readl(phba->HAregaddr, &ha_copy))
		goto unplug_err;

	if (ha_copy & HA_ERATT) {
		/* Read host status register to retrieve error event */
		if (lpfc_sli_read_hs(phba))
			goto unplug_err;

		/* Check if there is a deferred error condition is active */
		if ((HS_FFER1 & phba->work_hs) &&
		    ((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 |
		      HS_FFER6 | HS_FFER7 | HS_FFER8) & phba->work_hs)) {
			phba->hba_flag |= DEFER_ERATT;
			/* Clear all interrupt enable conditions */
			writel(0, phba->HCregaddr);
			readl(phba->HCregaddr);
		}

		/* Set the driver HA work bitmap */
		phba->work_ha |= HA_ERATT;
		/* Indicate polling handles this ERATT */
		phba->hba_flag |= HBA_ERATT_HANDLED;
		return 1;
	}
	return 0;

unplug_err:
	/* Set the driver HS work bitmap */
	phba->work_hs |= UNPLUG_ERR;
	/* Set the driver HA work bitmap */
	phba->work_ha |= HA_ERATT;
	/* Indicate polling handles this ERATT */
	phba->hba_flag |= HBA_ERATT_HANDLED;
	return 1;
}

/**
 * lpfc_sli4_eratt_read - read sli-4 error attention events
 * @phba: Pointer to HBA context.
 *
 * This function is called to read the SLI4 device error attention registers
 * for possible error attention events. The caller must hold the hostlock
 * with spin_lock_irq().
 *
 * This function returns 1 when there is Error Attention in the Host Attention
 * Register and returns 0 otherwise.
 **/
static int
lpfc_sli4_eratt_read(struct lpfc_hba *phba)
{
	uint32_t uerr_sta_hi, uerr_sta_lo;
	uint32_t if_type, portsmphr;
	struct lpfc_register portstat_reg;

	/*
	 * For now, use the SLI4 device internal unrecoverable error
	 * registers for error attention. This can be changed later.
	 */
	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
	switch (if_type) {
	case LPFC_SLI_INTF_IF_TYPE_0:
		if (lpfc_readl(phba->sli4_hba.u.if_type0.UERRLOregaddr,
			&uerr_sta_lo) ||
			lpfc_readl(phba->sli4_hba.u.if_type0.UERRHIregaddr,
			&uerr_sta_hi)) {
			phba->work_hs |= UNPLUG_ERR;
			phba->work_ha |= HA_ERATT;
			phba->hba_flag |= HBA_ERATT_HANDLED;
			return 1;
		}
		if ((~phba->sli4_hba.ue_mask_lo & uerr_sta_lo) ||
		    (~phba->sli4_hba.ue_mask_hi & uerr_sta_hi)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"1423 HBA Unrecoverable error: "
					"uerr_lo_reg=0x%x, uerr_hi_reg=0x%x, "
					"ue_mask_lo_reg=0x%x, "
					"ue_mask_hi_reg=0x%x\n",
					uerr_sta_lo, uerr_sta_hi,
					phba->sli4_hba.ue_mask_lo,
					phba->sli4_hba.ue_mask_hi);
			phba->work_status[0] = uerr_sta_lo;
			phba->work_status[1] = uerr_sta_hi;
			phba->work_ha |= HA_ERATT;
			phba->hba_flag |= HBA_ERATT_HANDLED;
			return 1;
		}
		break;
	case LPFC_SLI_INTF_IF_TYPE_2:
	case LPFC_SLI_INTF_IF_TYPE_6:
		if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr,
			&portstat_reg.word0) ||
			lpfc_readl(phba->sli4_hba.PSMPHRregaddr,
			&portsmphr)){
			phba->work_hs |= UNPLUG_ERR;
			phba->work_ha |= HA_ERATT;
			phba->hba_flag |= HBA_ERATT_HANDLED;
			return 1;
		}
		if (bf_get(lpfc_sliport_status_err, &portstat_reg)) {
			phba->work_status[0] =
				readl(phba->sli4_hba.u.if_type2.ERR1regaddr);
			phba->work_status[1] =
				readl(phba->sli4_hba.u.if_type2.ERR2regaddr);
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"2885 Port Status Event: "
					"port status reg 0x%x, "
					"port smphr reg 0x%x, "
					"error 1=0x%x, error 2=0x%x\n",
					portstat_reg.word0,
					portsmphr,
					phba->work_status[0],
					phba->work_status[1]);
			phba->work_ha |= HA_ERATT;
			phba->hba_flag |= HBA_ERATT_HANDLED;
			return 1;
		}
		break;
	case LPFC_SLI_INTF_IF_TYPE_1:
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2886 HBA Error Attention on unsupported "
				"if type %d.", if_type);
		return 1;
	}

	return 0;
}

/**
 * lpfc_sli_check_eratt - check error attention events
 * @phba: Pointer to HBA context.
 *
 * This function is called from timer soft interrupt context to check HBA's
 * error attention register bit for error attention events.
 *
 * This function returns 1 when there is Error Attention in the Host Attention
 * Register and returns 0 otherwise.
 **/
int
lpfc_sli_check_eratt(struct lpfc_hba *phba)
{
	uint32_t ha_copy;

	/* If somebody is waiting to handle an eratt, don't process it
	 * here. The brdkill function will do this.
	 */
	if (phba->link_flag & LS_IGNORE_ERATT)
		return 0;

	/* Check if interrupt handler handles this ERATT */
	spin_lock_irq(&phba->hbalock);
	if (phba->hba_flag & HBA_ERATT_HANDLED) {
		/* Interrupt handler has handled ERATT */
		spin_unlock_irq(&phba->hbalock);
		return 0;
	}

	/*
	 * If there is deferred error attention, do not check for error
	 * attention
	 */
	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
		spin_unlock_irq(&phba->hbalock);
		return 0;
	}

	/* If PCI channel is offline, don't process it */
	if (unlikely(pci_channel_offline(phba->pcidev))) {
		spin_unlock_irq(&phba->hbalock);
		return 0;
	}

	switch (phba->sli_rev) {
	case LPFC_SLI_REV2:
	case LPFC_SLI_REV3:
		/* Read chip Host Attention (HA) register */
		ha_copy = lpfc_sli_eratt_read(phba);
		break;
	case LPFC_SLI_REV4:
		/* Read device Uncoverable Error (UERR) registers */
		ha_copy = lpfc_sli4_eratt_read(phba);
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"0299 Invalid SLI revision (%d)\n",
				phba->sli_rev);
		ha_copy = 0;
		break;
	}
	spin_unlock_irq(&phba->hbalock);

	return ha_copy;
}

/**
 * lpfc_intr_state_check - Check device state for interrupt handling
 * @phba: Pointer to HBA context.
 *
 * This inline routine checks whether a device or its PCI slot is in a state
 * that the interrupt should be handled.
 *
 * This function returns 0 if the device or the PCI slot is in a state that
 * interrupt should be handled, otherwise -EIO.
 */
static inline int
lpfc_intr_state_check(struct lpfc_hba *phba)
{
	/* If the pci channel is offline, ignore all the interrupts */
	if (unlikely(pci_channel_offline(phba->pcidev)))
		return -EIO;

	/* Update device level interrupt statistics */
	phba->sli.slistat.sli_intr++;

	/* Ignore all interrupts during initialization. */
	if (unlikely(phba->link_state < LPFC_LINK_DOWN))
		return -EIO;

	return 0;
}

/**
 * lpfc_sli_sp_intr_handler - Slow-path interrupt handler to SLI-3 device
 * @irq: Interrupt number.
 * @dev_id: The device context pointer.
 *
 * This function is directly called from the PCI layer as an interrupt
 * service routine when device with SLI-3 interface spec is enabled with
 * MSI-X multi-message interrupt mode and there are slow-path events in
 * the HBA. However, when the device is enabled with either MSI or Pin-IRQ
 * interrupt mode, this function is called as part of the device-level
 * interrupt handler. When the PCI slot is in error recovery or the HBA
 * is undergoing initialization, the interrupt handler will not process
 * the interrupt. The link attention and ELS ring attention events are
 * handled by the worker thread. The interrupt handler signals the worker
 * thread and returns for these events. This function is called without
 * any lock held. It gets the hbalock to access and update SLI data
 * structures.
 *
 * This function returns IRQ_HANDLED when interrupt is handled else it
 * returns IRQ_NONE.
 **/
irqreturn_t
lpfc_sli_sp_intr_handler(int irq, void *dev_id)
{
	struct lpfc_hba  *phba;
	uint32_t ha_copy, hc_copy;
	uint32_t work_ha_copy;
	unsigned long status;
	unsigned long iflag;
	uint32_t control;

	MAILBOX_t *mbox, *pmbox;
	struct lpfc_vport *vport;
	struct lpfc_nodelist *ndlp;
	struct lpfc_dmabuf *mp;
	LPFC_MBOXQ_t *pmb;
	int rc;

	/*
	 * Get the driver's phba structure from the dev_id and
	 * assume the HBA is not interrupting.
	 */
	phba = (struct lpfc_hba *)dev_id;

	if (unlikely(!phba))
		return IRQ_NONE;

	/*
	 * Stuff needs to be attented to when this function is invoked as an
	 * individual interrupt handler in MSI-X multi-message interrupt mode
	 */
	if (phba->intr_type == MSIX) {
		/* Check device state for handling interrupt */
		if (lpfc_intr_state_check(phba))
			return IRQ_NONE;
		/* Need to read HA REG for slow-path events */
		spin_lock_irqsave(&phba->hbalock, iflag);
		if (lpfc_readl(phba->HAregaddr, &ha_copy))
			goto unplug_error;
		/* If somebody is waiting to handle an eratt don't process it
		 * here. The brdkill function will do this.
		 */
		if (phba->link_flag & LS_IGNORE_ERATT)
			ha_copy &= ~HA_ERATT;
		/* Check the need for handling ERATT in interrupt handler */
		if (ha_copy & HA_ERATT) {
			if (phba->hba_flag & HBA_ERATT_HANDLED)
				/* ERATT polling has handled ERATT */
				ha_copy &= ~HA_ERATT;
			else
				/* Indicate interrupt handler handles ERATT */
				phba->hba_flag |= HBA_ERATT_HANDLED;
		}

		/*
		 * If there is deferred error attention, do not check for any
		 * interrupt.
		 */
		if (unlikely(phba->hba_flag & DEFER_ERATT)) {
			spin_unlock_irqrestore(&phba->hbalock, iflag);
			return IRQ_NONE;
		}

		/* Clear up only attention source related to slow-path */
		if (lpfc_readl(phba->HCregaddr, &hc_copy))
			goto unplug_error;

		writel(hc_copy & ~(HC_MBINT_ENA | HC_R2INT_ENA |
			HC_LAINT_ENA | HC_ERINT_ENA),
			phba->HCregaddr);
		writel((ha_copy & (HA_MBATT | HA_R2_CLR_MSK)),
			phba->HAregaddr);
		writel(hc_copy, phba->HCregaddr);
		readl(phba->HAregaddr); /* flush */
		spin_unlock_irqrestore(&phba->hbalock, iflag);
	} else
		ha_copy = phba->ha_copy;

	work_ha_copy = ha_copy & phba->work_ha_mask;

	if (work_ha_copy) {
		if (work_ha_copy & HA_LATT) {
			if (phba->sli.sli_flag & LPFC_PROCESS_LA) {
				/*
				 * Turn off Link Attention interrupts
				 * until CLEAR_LA done
				 */
				spin_lock_irqsave(&phba->hbalock, iflag);
				phba->sli.sli_flag &= ~LPFC_PROCESS_LA;
				if (lpfc_readl(phba->HCregaddr, &control))
					goto unplug_error;
				control &= ~HC_LAINT_ENA;
				writel(control, phba->HCregaddr);
				readl(phba->HCregaddr); /* flush */
				spin_unlock_irqrestore(&phba->hbalock, iflag);
			}
			else
				work_ha_copy &= ~HA_LATT;
		}

		if (work_ha_copy & ~(HA_ERATT | HA_MBATT | HA_LATT)) {
			/*
			 * Turn off Slow Rings interrupts, LPFC_ELS_RING is
			 * the only slow ring.
			 */
			status = (work_ha_copy &
				(HA_RXMASK  << (4*LPFC_ELS_RING)));
			status >>= (4*LPFC_ELS_RING);
			if (status & HA_RXMASK) {
				spin_lock_irqsave(&phba->hbalock, iflag);
				if (lpfc_readl(phba->HCregaddr, &control))
					goto unplug_error;

				lpfc_debugfs_slow_ring_trc(phba,
				"ISR slow ring:   ctl:x%x stat:x%x isrcnt:x%x",
				control, status,
				(uint32_t)phba->sli.slistat.sli_intr);

				if (control & (HC_R0INT_ENA << LPFC_ELS_RING)) {
					lpfc_debugfs_slow_ring_trc(phba,
						"ISR Disable ring:"
						"pwork:x%x hawork:x%x wait:x%x",
						phba->work_ha, work_ha_copy,
						(uint32_t)((unsigned long)
						&phba->work_waitq));

					control &=
					    ~(HC_R0INT_ENA << LPFC_ELS_RING);
					writel(control, phba->HCregaddr);
					readl(phba->HCregaddr); /* flush */
				}
				else {
					lpfc_debugfs_slow_ring_trc(phba,
						"ISR slow ring:   pwork:"
						"x%x hawork:x%x wait:x%x",
						phba->work_ha, work_ha_copy,
						(uint32_t)((unsigned long)
						&phba->work_waitq));
				}
				spin_unlock_irqrestore(&phba->hbalock, iflag);
			}
		}
		spin_lock_irqsave(&phba->hbalock, iflag);
		if (work_ha_copy & HA_ERATT) {
			if (lpfc_sli_read_hs(phba))
				goto unplug_error;
			/*
			 * Check if there is a deferred error condition
			 * is active
			 */
			if ((HS_FFER1 & phba->work_hs) &&
				((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 |
				  HS_FFER6 | HS_FFER7 | HS_FFER8) &
				  phba->work_hs)) {
				phba->hba_flag |= DEFER_ERATT;
				/* Clear all interrupt enable conditions */
				writel(0, phba->HCregaddr);
				readl(phba->HCregaddr);
			}
		}

		if ((work_ha_copy & HA_MBATT) && (phba->sli.mbox_active)) {
			pmb = phba->sli.mbox_active;
			pmbox = &pmb->u.mb;
			mbox = phba->mbox;
			vport = pmb->vport;

			/* First check out the status word */
			lpfc_sli_pcimem_bcopy(mbox, pmbox, sizeof(uint32_t));
			if (pmbox->mbxOwner != OWN_HOST) {
				spin_unlock_irqrestore(&phba->hbalock, iflag);
				/*
				 * Stray Mailbox Interrupt, mbxCommand <cmd>
				 * mbxStatus <status>
				 */
				lpfc_printf_log(phba, KERN_ERR, LOG_MBOX |
						LOG_SLI,
						"(%d):0304 Stray Mailbox "
						"Interrupt mbxCommand x%x "
						"mbxStatus x%x\n",
						(vport ? vport->vpi : 0),
						pmbox->mbxCommand,
						pmbox->mbxStatus);
				/* clear mailbox attention bit */
				work_ha_copy &= ~HA_MBATT;
			} else {
				phba->sli.mbox_active = NULL;
				spin_unlock_irqrestore(&phba->hbalock, iflag);
				phba->last_completion_time = jiffies;
				del_timer(&phba->sli.mbox_tmo);
				if (pmb->mbox_cmpl) {
					lpfc_sli_pcimem_bcopy(mbox, pmbox,
							MAILBOX_CMD_SIZE);
					if (pmb->out_ext_byte_len &&
						pmb->ctx_buf)
						lpfc_sli_pcimem_bcopy(
						phba->mbox_ext,
						pmb->ctx_buf,
						pmb->out_ext_byte_len);
				}
				if (pmb->mbox_flag & LPFC_MBX_IMED_UNREG) {
					pmb->mbox_flag &= ~LPFC_MBX_IMED_UNREG;

					lpfc_debugfs_disc_trc(vport,
						LPFC_DISC_TRC_MBOX_VPORT,
						"MBOX dflt rpi: : "
						"status:x%x rpi:x%x",
						(uint32_t)pmbox->mbxStatus,
						pmbox->un.varWords[0], 0);

					if (!pmbox->mbxStatus) {
						mp = (struct lpfc_dmabuf *)
							(pmb->ctx_buf);
						ndlp = (struct lpfc_nodelist *)
							pmb->ctx_ndlp;

						/* Reg_LOGIN of dflt RPI was
						 * successful. new lets get
						 * rid of the RPI using the
						 * same mbox buffer.
						 */
						lpfc_unreg_login(phba,
							vport->vpi,
							pmbox->un.varWords[0],
							pmb);
						pmb->mbox_cmpl =
							lpfc_mbx_cmpl_dflt_rpi;
						pmb->ctx_buf = mp;
						pmb->ctx_ndlp = ndlp;
						pmb->vport = vport;
						rc = lpfc_sli_issue_mbox(phba,
								pmb,
								MBX_NOWAIT);
						if (rc != MBX_BUSY)
							lpfc_printf_log(phba,
							KERN_ERR,
							LOG_MBOX | LOG_SLI,
							"0350 rc should have"
							"been MBX_BUSY\n");
						if (rc != MBX_NOT_FINISHED)
							goto send_current_mbox;
					}
				}
				spin_lock_irqsave(
						&phba->pport->work_port_lock,
						iflag);
				phba->pport->work_port_events &=
					~WORKER_MBOX_TMO;
				spin_unlock_irqrestore(
						&phba->pport->work_port_lock,
						iflag);
				lpfc_mbox_cmpl_put(phba, pmb);
			}
		} else
			spin_unlock_irqrestore(&phba->hbalock, iflag);

		if ((work_ha_copy & HA_MBATT) &&
		    (phba->sli.mbox_active == NULL)) {
send_current_mbox:
			/* Process next mailbox command if there is one */
			do {
				rc = lpfc_sli_issue_mbox(phba, NULL,
							 MBX_NOWAIT);
			} while (rc == MBX_NOT_FINISHED);
			if (rc != MBX_SUCCESS)
				lpfc_printf_log(phba, KERN_ERR, LOG_MBOX |
						LOG_SLI, "0349 rc should be "
						"MBX_SUCCESS\n");
		}

		spin_lock_irqsave(&phba->hbalock, iflag);
		phba->work_ha |= work_ha_copy;
		spin_unlock_irqrestore(&phba->hbalock, iflag);
		lpfc_worker_wake_up(phba);
	}
	return IRQ_HANDLED;
unplug_error:
	spin_unlock_irqrestore(&phba->hbalock, iflag);
	return IRQ_HANDLED;

} /* lpfc_sli_sp_intr_handler */

/**
 * lpfc_sli_fp_intr_handler - Fast-path interrupt handler to SLI-3 device.
 * @irq: Interrupt number.
 * @dev_id: The device context pointer.
 *
 * This function is directly called from the PCI layer as an interrupt
 * service routine when device with SLI-3 interface spec is enabled with
 * MSI-X multi-message interrupt mode and there is a fast-path FCP IOCB
 * ring event in the HBA. However, when the device is enabled with either
 * MSI or Pin-IRQ interrupt mode, this function is called as part of the
 * device-level interrupt handler. When the PCI slot is in error recovery
 * or the HBA is undergoing initialization, the interrupt handler will not
 * process the interrupt. The SCSI FCP fast-path ring event are handled in
 * the intrrupt context. This function is called without any lock held.
 * It gets the hbalock to access and update SLI data structures.
 *
 * This function returns IRQ_HANDLED when interrupt is handled else it
 * returns IRQ_NONE.
 **/
irqreturn_t
lpfc_sli_fp_intr_handler(int irq, void *dev_id)
{
	struct lpfc_hba  *phba;
	uint32_t ha_copy;
	unsigned long status;
	unsigned long iflag;
	struct lpfc_sli_ring *pring;

	/* Get the driver's phba structure from the dev_id and
	 * assume the HBA is not interrupting.
	 */
	phba = (struct lpfc_hba *) dev_id;

	if (unlikely(!phba))
		return IRQ_NONE;

	/*
	 * Stuff needs to be attented to when this function is invoked as an
	 * individual interrupt handler in MSI-X multi-message interrupt mode
	 */
	if (phba->intr_type == MSIX) {
		/* Check device state for handling interrupt */
		if (lpfc_intr_state_check(phba))
			return IRQ_NONE;
		/* Need to read HA REG for FCP ring and other ring events */
		if (lpfc_readl(phba->HAregaddr, &ha_copy))
			return IRQ_HANDLED;
		/* Clear up only attention source related to fast-path */
		spin_lock_irqsave(&phba->hbalock, iflag);
		/*
		 * If there is deferred error attention, do not check for
		 * any interrupt.
		 */
		if (unlikely(phba->hba_flag & DEFER_ERATT)) {
			spin_unlock_irqrestore(&phba->hbalock, iflag);
			return IRQ_NONE;
		}
		writel((ha_copy & (HA_R0_CLR_MSK | HA_R1_CLR_MSK)),
			phba->HAregaddr);
		readl(phba->HAregaddr); /* flush */
		spin_unlock_irqrestore(&phba->hbalock, iflag);
	} else
		ha_copy = phba->ha_copy;

	/*
	 * Process all events on FCP ring. Take the optimized path for FCP IO.
	 */
	ha_copy &= ~(phba->work_ha_mask);

	status = (ha_copy & (HA_RXMASK << (4*LPFC_FCP_RING)));
	status >>= (4*LPFC_FCP_RING);
	pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
	if (status & HA_RXMASK)
		lpfc_sli_handle_fast_ring_event(phba, pring, status);

	if (phba->cfg_multi_ring_support == 2) {
		/*
		 * Process all events on extra ring. Take the optimized path
		 * for extra ring IO.
		 */
		status = (ha_copy & (HA_RXMASK << (4*LPFC_EXTRA_RING)));
		status >>= (4*LPFC_EXTRA_RING);
		if (status & HA_RXMASK) {
			lpfc_sli_handle_fast_ring_event(phba,
					&phba->sli.sli3_ring[LPFC_EXTRA_RING],
					status);
		}
	}
	return IRQ_HANDLED;
}  /* lpfc_sli_fp_intr_handler */

/**
 * lpfc_sli_intr_handler - Device-level interrupt handler to SLI-3 device
 * @irq: Interrupt number.
 * @dev_id: The device context pointer.
 *
 * This function is the HBA device-level interrupt handler to device with
 * SLI-3 interface spec, called from the PCI layer when either MSI or
 * Pin-IRQ interrupt mode is enabled and there is an event in the HBA which
 * requires driver attention. This function invokes the slow-path interrupt
 * attention handling function and fast-path interrupt attention handling
 * function in turn to process the relevant HBA attention events. This
 * function is called without any lock held. It gets the hbalock to access
 * and update SLI data structures.
 *
 * This function returns IRQ_HANDLED when interrupt is handled, else it
 * returns IRQ_NONE.
 **/
irqreturn_t
lpfc_sli_intr_handler(int irq, void *dev_id)
{
	struct lpfc_hba  *phba;
	irqreturn_t sp_irq_rc, fp_irq_rc;
	unsigned long status1, status2;
	uint32_t hc_copy;

	/*
	 * Get the driver's phba structure from the dev_id and
	 * assume the HBA is not interrupting.
	 */
	phba = (struct lpfc_hba *) dev_id;

	if (unlikely(!phba))
		return IRQ_NONE;

	/* Check device state for handling interrupt */
	if (lpfc_intr_state_check(phba))
		return IRQ_NONE;

	spin_lock(&phba->hbalock);
	if (lpfc_readl(phba->HAregaddr, &phba->ha_copy)) {
		spin_unlock(&phba->hbalock);
		return IRQ_HANDLED;
	}

	if (unlikely(!phba->ha_copy)) {
		spin_unlock(&phba->hbalock);
		return IRQ_NONE;
	} else if (phba->ha_copy & HA_ERATT) {
		if (phba->hba_flag & HBA_ERATT_HANDLED)
			/* ERATT polling has handled ERATT */
			phba->ha_copy &= ~HA_ERATT;
		else
			/* Indicate interrupt handler handles ERATT */
			phba->hba_flag |= HBA_ERATT_HANDLED;
	}

	/*
	 * If there is deferred error attention, do not check for any interrupt.
	 */
	if (unlikely(phba->hba_flag & DEFER_ERATT)) {
		spin_unlock(&phba->hbalock);
		return IRQ_NONE;
	}

	/* Clear attention sources except link and error attentions */
	if (lpfc_readl(phba->HCregaddr, &hc_copy)) {
		spin_unlock(&phba->hbalock);
		return IRQ_HANDLED;
	}
	writel(hc_copy & ~(HC_MBINT_ENA | HC_R0INT_ENA | HC_R1INT_ENA
		| HC_R2INT_ENA | HC_LAINT_ENA | HC_ERINT_ENA),
		phba->HCregaddr);
	writel((phba->ha_copy & ~(HA_LATT | HA_ERATT)), phba->HAregaddr);
	writel(hc_copy, phba->HCregaddr);
	readl(phba->HAregaddr); /* flush */
	spin_unlock(&phba->hbalock);

	/*
	 * Invokes slow-path host attention interrupt handling as appropriate.
	 */

	/* status of events with mailbox and link attention */
	status1 = phba->ha_copy & (HA_MBATT | HA_LATT | HA_ERATT);

	/* status of events with ELS ring */
	status2 = (phba->ha_copy & (HA_RXMASK  << (4*LPFC_ELS_RING)));
	status2 >>= (4*LPFC_ELS_RING);

	if (status1 || (status2 & HA_RXMASK))
		sp_irq_rc = lpfc_sli_sp_intr_handler(irq, dev_id);
	else
		sp_irq_rc = IRQ_NONE;

	/*
	 * Invoke fast-path host attention interrupt handling as appropriate.
	 */

	/* status of events with FCP ring */
	status1 = (phba->ha_copy & (HA_RXMASK << (4*LPFC_FCP_RING)));
	status1 >>= (4*LPFC_FCP_RING);

	/* status of events with extra ring */
	if (phba->cfg_multi_ring_support == 2) {
		status2 = (phba->ha_copy & (HA_RXMASK << (4*LPFC_EXTRA_RING)));
		status2 >>= (4*LPFC_EXTRA_RING);
	} else
		status2 = 0;

	if ((status1 & HA_RXMASK) || (status2 & HA_RXMASK))
		fp_irq_rc = lpfc_sli_fp_intr_handler(irq, dev_id);
	else
		fp_irq_rc = IRQ_NONE;

	/* Return device-level interrupt handling status */
	return (sp_irq_rc == IRQ_HANDLED) ? sp_irq_rc : fp_irq_rc;
}  /* lpfc_sli_intr_handler */

/**
 * lpfc_sli4_els_xri_abort_event_proc - Process els xri abort event
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine is invoked by the worker thread to process all the pending
 * SLI4 els abort xri events.
 **/
void lpfc_sli4_els_xri_abort_event_proc(struct lpfc_hba *phba)
{
	struct lpfc_cq_event *cq_event;

	/* First, declare the els xri abort event has been handled */
	spin_lock_irq(&phba->hbalock);
	phba->hba_flag &= ~ELS_XRI_ABORT_EVENT;
	spin_unlock_irq(&phba->hbalock);
	/* Now, handle all the els xri abort events */
	while (!list_empty(&phba->sli4_hba.sp_els_xri_aborted_work_queue)) {
		/* Get the first event from the head of the event queue */
		spin_lock_irq(&phba->hbalock);
		list_remove_head(&phba->sli4_hba.sp_els_xri_aborted_work_queue,
				 cq_event, struct lpfc_cq_event, list);
		spin_unlock_irq(&phba->hbalock);
		/* Notify aborted XRI for ELS work queue */
		lpfc_sli4_els_xri_aborted(phba, &cq_event->cqe.wcqe_axri);
		/* Free the event processed back to the free pool */
		lpfc_sli4_cq_event_release(phba, cq_event);
	}
}

/**
 * lpfc_sli4_iocb_param_transfer - Transfer pIocbOut and cmpl status to pIocbIn
 * @phba: pointer to lpfc hba data structure
 * @pIocbIn: pointer to the rspiocbq
 * @pIocbOut: pointer to the cmdiocbq
 * @wcqe: pointer to the complete wcqe
 *
 * This routine transfers the fields of a command iocbq to a response iocbq
 * by copying all the IOCB fields from command iocbq and transferring the
 * completion status information from the complete wcqe.
 **/
static void
lpfc_sli4_iocb_param_transfer(struct lpfc_hba *phba,
			      struct lpfc_iocbq *pIocbIn,
			      struct lpfc_iocbq *pIocbOut,
			      struct lpfc_wcqe_complete *wcqe)
{
	int numBdes, i;
	unsigned long iflags;
	uint32_t status, max_response;
	struct lpfc_dmabuf *dmabuf;
	struct ulp_bde64 *bpl, bde;
	size_t offset = offsetof(struct lpfc_iocbq, iocb);

	memcpy((char *)pIocbIn + offset, (char *)pIocbOut + offset,
	       sizeof(struct lpfc_iocbq) - offset);
	/* Map WCQE parameters into irspiocb parameters */
	status = bf_get(lpfc_wcqe_c_status, wcqe);
	pIocbIn->iocb.ulpStatus = (status & LPFC_IOCB_STATUS_MASK);
	if (pIocbOut->iocb_flag & LPFC_IO_FCP)
		if (pIocbIn->iocb.ulpStatus == IOSTAT_FCP_RSP_ERROR)
			pIocbIn->iocb.un.fcpi.fcpi_parm =
					pIocbOut->iocb.un.fcpi.fcpi_parm -
					wcqe->total_data_placed;
		else
			pIocbIn->iocb.un.ulpWord[4] = wcqe->parameter;
	else {
		pIocbIn->iocb.un.ulpWord[4] = wcqe->parameter;
		switch (pIocbOut->iocb.ulpCommand) {
		case CMD_ELS_REQUEST64_CR:
			dmabuf = (struct lpfc_dmabuf *)pIocbOut->context3;
			bpl  = (struct ulp_bde64 *)dmabuf->virt;
			bde.tus.w = le32_to_cpu(bpl[1].tus.w);
			max_response = bde.tus.f.bdeSize;
			break;
		case CMD_GEN_REQUEST64_CR:
			max_response = 0;
			if (!pIocbOut->context3)
				break;
			numBdes = pIocbOut->iocb.un.genreq64.bdl.bdeSize/
					sizeof(struct ulp_bde64);
			dmabuf = (struct lpfc_dmabuf *)pIocbOut->context3;
			bpl = (struct ulp_bde64 *)dmabuf->virt;
			for (i = 0; i < numBdes; i++) {
				bde.tus.w = le32_to_cpu(bpl[i].tus.w);
				if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
					max_response += bde.tus.f.bdeSize;
			}
			break;
		default:
			max_response = wcqe->total_data_placed;
			break;
		}
		if (max_response < wcqe->total_data_placed)
			pIocbIn->iocb.un.genreq64.bdl.bdeSize = max_response;
		else
			pIocbIn->iocb.un.genreq64.bdl.bdeSize =
				wcqe->total_data_placed;
	}

	/* Convert BG errors for completion status */
	if (status == CQE_STATUS_DI_ERROR) {
		pIocbIn->iocb.ulpStatus = IOSTAT_LOCAL_REJECT;

		if (bf_get(lpfc_wcqe_c_bg_edir, wcqe))
			pIocbIn->iocb.un.ulpWord[4] = IOERR_RX_DMA_FAILED;
		else
			pIocbIn->iocb.un.ulpWord[4] = IOERR_TX_DMA_FAILED;

		pIocbIn->iocb.unsli3.sli3_bg.bgstat = 0;
		if (bf_get(lpfc_wcqe_c_bg_ge, wcqe)) /* Guard Check failed */
			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
				BGS_GUARD_ERR_MASK;
		if (bf_get(lpfc_wcqe_c_bg_ae, wcqe)) /* App Tag Check failed */
			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
				BGS_APPTAG_ERR_MASK;
		if (bf_get(lpfc_wcqe_c_bg_re, wcqe)) /* Ref Tag Check failed */
			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
				BGS_REFTAG_ERR_MASK;

		/* Check to see if there was any good data before the error */
		if (bf_get(lpfc_wcqe_c_bg_tdpv, wcqe)) {
			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
				BGS_HI_WATER_MARK_PRESENT_MASK;
			pIocbIn->iocb.unsli3.sli3_bg.bghm =
				wcqe->total_data_placed;
		}

		/*
		* Set ALL the error bits to indicate we don't know what
		* type of error it is.
		*/
		if (!pIocbIn->iocb.unsli3.sli3_bg.bgstat)
			pIocbIn->iocb.unsli3.sli3_bg.bgstat |=
				(BGS_REFTAG_ERR_MASK | BGS_APPTAG_ERR_MASK |
				BGS_GUARD_ERR_MASK);
	}

	/* Pick up HBA exchange busy condition */
	if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
		spin_lock_irqsave(&phba->hbalock, iflags);
		pIocbIn->iocb_flag |= LPFC_EXCHANGE_BUSY;
		spin_unlock_irqrestore(&phba->hbalock, iflags);
	}
}

/**
 * lpfc_sli4_els_wcqe_to_rspiocbq - Get response iocbq from els wcqe
 * @phba: Pointer to HBA context object.
 * @wcqe: Pointer to work-queue completion queue entry.
 *
 * This routine handles an ELS work-queue completion event and construct
 * a pseudo response ELS IODBQ from the SLI4 ELS WCQE for the common
 * discovery engine to handle.
 *
 * Return: Pointer to the receive IOCBQ, NULL otherwise.
 **/
static struct lpfc_iocbq *
lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba,
			       struct lpfc_iocbq *irspiocbq)
{
	struct lpfc_sli_ring *pring;
	struct lpfc_iocbq *cmdiocbq;
	struct lpfc_wcqe_complete *wcqe;
	unsigned long iflags;

	pring = lpfc_phba_elsring(phba);
	if (unlikely(!pring))
		return NULL;

	wcqe = &irspiocbq->cq_event.cqe.wcqe_cmpl;
	pring->stats.iocb_event++;
	/* Look up the ELS command IOCB and create pseudo response IOCB */
	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
				bf_get(lpfc_wcqe_c_request_tag, wcqe));
	if (unlikely(!cmdiocbq)) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"0386 ELS complete with no corresponding "
				"cmdiocb: 0x%x 0x%x 0x%x 0x%x\n",
				wcqe->word0, wcqe->total_data_placed,
				wcqe->parameter, wcqe->word3);
		lpfc_sli_release_iocbq(phba, irspiocbq);
		return NULL;
	}

	spin_lock_irqsave(&pring->ring_lock, iflags);
	/* Put the iocb back on the txcmplq */
	lpfc_sli_ringtxcmpl_put(phba, pring, cmdiocbq);
	spin_unlock_irqrestore(&pring->ring_lock, iflags);

	/* Fake the irspiocbq and copy necessary response information */
	lpfc_sli4_iocb_param_transfer(phba, irspiocbq, cmdiocbq, wcqe);

	return irspiocbq;
}

inline struct lpfc_cq_event *
lpfc_cq_event_setup(struct lpfc_hba *phba, void *entry, int size)
{
	struct lpfc_cq_event *cq_event;

	/* Allocate a new internal CQ_EVENT entry */
	cq_event = lpfc_sli4_cq_event_alloc(phba);
	if (!cq_event) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0602 Failed to alloc CQ_EVENT entry\n");
		return NULL;
	}

	/* Move the CQE into the event */
	memcpy(&cq_event->cqe, entry, size);
	return cq_event;
}

/**
 * lpfc_sli4_sp_handle_async_event - Handle an asynchroous event
 * @phba: Pointer to HBA context object.
 * @cqe: Pointer to mailbox completion queue entry.
 *
 * This routine process a mailbox completion queue entry with asynchrous
 * event.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_sp_handle_async_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
{
	struct lpfc_cq_event *cq_event;
	unsigned long iflags;

	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
			"0392 Async Event: word0:x%x, word1:x%x, "
			"word2:x%x, word3:x%x\n", mcqe->word0,
			mcqe->mcqe_tag0, mcqe->mcqe_tag1, mcqe->trailer);

	cq_event = lpfc_cq_event_setup(phba, mcqe, sizeof(struct lpfc_mcqe));
	if (!cq_event)
		return false;
	spin_lock_irqsave(&phba->hbalock, iflags);
	list_add_tail(&cq_event->list, &phba->sli4_hba.sp_asynce_work_queue);
	/* Set the async event flag */
	phba->hba_flag |= ASYNC_EVENT;
	spin_unlock_irqrestore(&phba->hbalock, iflags);

	return true;
}

/**
 * lpfc_sli4_sp_handle_mbox_event - Handle a mailbox completion event
 * @phba: Pointer to HBA context object.
 * @cqe: Pointer to mailbox completion queue entry.
 *
 * This routine process a mailbox completion queue entry with mailbox
 * completion event.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
{
	uint32_t mcqe_status;
	MAILBOX_t *mbox, *pmbox;
	struct lpfc_mqe *mqe;
	struct lpfc_vport *vport;
	struct lpfc_nodelist *ndlp;
	struct lpfc_dmabuf *mp;
	unsigned long iflags;
	LPFC_MBOXQ_t *pmb;
	bool workposted = false;
	int rc;

	/* If not a mailbox complete MCQE, out by checking mailbox consume */
	if (!bf_get(lpfc_trailer_completed, mcqe))
		goto out_no_mqe_complete;

	/* Get the reference to the active mbox command */
	spin_lock_irqsave(&phba->hbalock, iflags);
	pmb = phba->sli.mbox_active;
	if (unlikely(!pmb)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_MBOX,
				"1832 No pending MBOX command to handle\n");
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		goto out_no_mqe_complete;
	}
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	mqe = &pmb->u.mqe;
	pmbox = (MAILBOX_t *)&pmb->u.mqe;
	mbox = phba->mbox;
	vport = pmb->vport;

	/* Reset heartbeat timer */
	phba->last_completion_time = jiffies;
	del_timer(&phba->sli.mbox_tmo);

	/* Move mbox data to caller's mailbox region, do endian swapping */
	if (pmb->mbox_cmpl && mbox)
		lpfc_sli4_pcimem_bcopy(mbox, mqe, sizeof(struct lpfc_mqe));

	/*
	 * For mcqe errors, conditionally move a modified error code to
	 * the mbox so that the error will not be missed.
	 */
	mcqe_status = bf_get(lpfc_mcqe_status, mcqe);
	if (mcqe_status != MB_CQE_STATUS_SUCCESS) {
		if (bf_get(lpfc_mqe_status, mqe) == MBX_SUCCESS)
			bf_set(lpfc_mqe_status, mqe,
			       (LPFC_MBX_ERROR_RANGE | mcqe_status));
	}
	if (pmb->mbox_flag & LPFC_MBX_IMED_UNREG) {
		pmb->mbox_flag &= ~LPFC_MBX_IMED_UNREG;
		lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_MBOX_VPORT,
				      "MBOX dflt rpi: status:x%x rpi:x%x",
				      mcqe_status,
				      pmbox->un.varWords[0], 0);
		if (mcqe_status == MB_CQE_STATUS_SUCCESS) {
			mp = (struct lpfc_dmabuf *)(pmb->ctx_buf);
			ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp;
			/* Reg_LOGIN of dflt RPI was successful. Now lets get
			 * RID of the PPI using the same mbox buffer.
			 */
			lpfc_unreg_login(phba, vport->vpi,
					 pmbox->un.varWords[0], pmb);
			pmb->mbox_cmpl = lpfc_mbx_cmpl_dflt_rpi;
			pmb->ctx_buf = mp;
			pmb->ctx_ndlp = ndlp;
			pmb->vport = vport;
			rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
			if (rc != MBX_BUSY)
				lpfc_printf_log(phba, KERN_ERR, LOG_MBOX |
						LOG_SLI, "0385 rc should "
						"have been MBX_BUSY\n");
			if (rc != MBX_NOT_FINISHED)
				goto send_current_mbox;
		}
	}
	spin_lock_irqsave(&phba->pport->work_port_lock, iflags);
	phba->pport->work_port_events &= ~WORKER_MBOX_TMO;
	spin_unlock_irqrestore(&phba->pport->work_port_lock, iflags);

	/* There is mailbox completion work to do */
	spin_lock_irqsave(&phba->hbalock, iflags);
	__lpfc_mbox_cmpl_put(phba, pmb);
	phba->work_ha |= HA_MBATT;
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	workposted = true;

send_current_mbox:
	spin_lock_irqsave(&phba->hbalock, iflags);
	/* Release the mailbox command posting token */
	phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE;
	/* Setting active mailbox pointer need to be in sync to flag clear */
	phba->sli.mbox_active = NULL;
	spin_unlock_irqrestore(&phba->hbalock, iflags);
	/* Wake up worker thread to post the next pending mailbox command */
	lpfc_worker_wake_up(phba);
out_no_mqe_complete:
	if (bf_get(lpfc_trailer_consumed, mcqe))
		lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq);
	return workposted;
}

/**
 * lpfc_sli4_sp_handle_mcqe - Process a mailbox completion queue entry
 * @phba: Pointer to HBA context object.
 * @cqe: Pointer to mailbox completion queue entry.
 *
 * This routine process a mailbox completion queue entry, it invokes the
 * proper mailbox complete handling or asynchrous event handling routine
 * according to the MCQE's async bit.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			 struct lpfc_cqe *cqe)
{
	struct lpfc_mcqe mcqe;
	bool workposted;

	cq->CQ_mbox++;

	/* Copy the mailbox MCQE and convert endian order as needed */
	lpfc_sli4_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe));

	/* Invoke the proper event handling routine */
	if (!bf_get(lpfc_trailer_async, &mcqe))
		workposted = lpfc_sli4_sp_handle_mbox_event(phba, &mcqe);
	else
		workposted = lpfc_sli4_sp_handle_async_event(phba, &mcqe);
	return workposted;
}

/**
 * lpfc_sli4_sp_handle_els_wcqe - Handle els work-queue completion event
 * @phba: Pointer to HBA context object.
 * @cq: Pointer to associated CQ
 * @wcqe: Pointer to work-queue completion queue entry.
 *
 * This routine handles an ELS work-queue completion event.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			     struct lpfc_wcqe_complete *wcqe)
{
	struct lpfc_iocbq *irspiocbq;
	unsigned long iflags;
	struct lpfc_sli_ring *pring = cq->pring;
	int txq_cnt = 0;
	int txcmplq_cnt = 0;
	int fcp_txcmplq_cnt = 0;

	/* Check for response status */
	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
		/* Log the error status */
		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"0357 ELS CQE error: status=x%x: "
				"CQE: %08x %08x %08x %08x\n",
				bf_get(lpfc_wcqe_c_status, wcqe),
				wcqe->word0, wcqe->total_data_placed,
				wcqe->parameter, wcqe->word3);
	}

	/* Get an irspiocbq for later ELS response processing use */
	irspiocbq = lpfc_sli_get_iocbq(phba);
	if (!irspiocbq) {
		if (!list_empty(&pring->txq))
			txq_cnt++;
		if (!list_empty(&pring->txcmplq))
			txcmplq_cnt++;
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
			"0387 NO IOCBQ data: txq_cnt=%d iocb_cnt=%d "
			"fcp_txcmplq_cnt=%d, els_txcmplq_cnt=%d\n",
			txq_cnt, phba->iocb_cnt,
			fcp_txcmplq_cnt,
			txcmplq_cnt);
		return false;
	}

	/* Save off the slow-path queue event for work thread to process */
	memcpy(&irspiocbq->cq_event.cqe.wcqe_cmpl, wcqe, sizeof(*wcqe));
	spin_lock_irqsave(&phba->hbalock, iflags);
	list_add_tail(&irspiocbq->cq_event.list,
		      &phba->sli4_hba.sp_queue_event);
	phba->hba_flag |= HBA_SP_QUEUE_EVT;
	spin_unlock_irqrestore(&phba->hbalock, iflags);

	return true;
}

/**
 * lpfc_sli4_sp_handle_rel_wcqe - Handle slow-path WQ entry consumed event
 * @phba: Pointer to HBA context object.
 * @wcqe: Pointer to work-queue completion queue entry.
 *
 * This routine handles slow-path WQ entry consumed event by invoking the
 * proper WQ release routine to the slow-path WQ.
 **/
static void
lpfc_sli4_sp_handle_rel_wcqe(struct lpfc_hba *phba,
			     struct lpfc_wcqe_release *wcqe)
{
	/* sanity check on queue memory */
	if (unlikely(!phba->sli4_hba.els_wq))
		return;
	/* Check for the slow-path ELS work queue */
	if (bf_get(lpfc_wcqe_r_wq_id, wcqe) == phba->sli4_hba.els_wq->queue_id)
		lpfc_sli4_wq_release(phba->sli4_hba.els_wq,
				     bf_get(lpfc_wcqe_r_wqe_index, wcqe));
	else
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"2579 Slow-path wqe consume event carries "
				"miss-matched qid: wcqe-qid=x%x, sp-qid=x%x\n",
				bf_get(lpfc_wcqe_r_wqe_index, wcqe),
				phba->sli4_hba.els_wq->queue_id);
}

/**
 * lpfc_sli4_sp_handle_abort_xri_wcqe - Handle a xri abort event
 * @phba: Pointer to HBA context object.
 * @cq: Pointer to a WQ completion queue.
 * @wcqe: Pointer to work-queue completion queue entry.
 *
 * This routine handles an XRI abort event.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba,
				   struct lpfc_queue *cq,
				   struct sli4_wcqe_xri_aborted *wcqe)
{
	bool workposted = false;
	struct lpfc_cq_event *cq_event;
	unsigned long iflags;

	switch (cq->subtype) {
	case LPFC_FCP:
		lpfc_sli4_fcp_xri_aborted(phba, wcqe, cq->hdwq);
		workposted = false;
		break;
	case LPFC_NVME_LS: /* NVME LS uses ELS resources */
	case LPFC_ELS:
		cq_event = lpfc_cq_event_setup(
			phba, wcqe, sizeof(struct sli4_wcqe_xri_aborted));
		if (!cq_event)
			return false;
		cq_event->hdwq = cq->hdwq;
		spin_lock_irqsave(&phba->hbalock, iflags);
		list_add_tail(&cq_event->list,
			      &phba->sli4_hba.sp_els_xri_aborted_work_queue);
		/* Set the els xri abort event flag */
		phba->hba_flag |= ELS_XRI_ABORT_EVENT;
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		workposted = true;
		break;
	case LPFC_NVME:
		/* Notify aborted XRI for NVME work queue */
		if (phba->nvmet_support)
			lpfc_sli4_nvmet_xri_aborted(phba, wcqe);
		else
			lpfc_sli4_nvme_xri_aborted(phba, wcqe, cq->hdwq);

		workposted = false;
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0603 Invalid CQ subtype %d: "
				"%08x %08x %08x %08x\n",
				cq->subtype, wcqe->word0, wcqe->parameter,
				wcqe->word2, wcqe->word3);
		workposted = false;
		break;
	}
	return workposted;
}

#define FC_RCTL_MDS_DIAGS	0xF4

/**
 * lpfc_sli4_sp_handle_rcqe - Process a receive-queue completion queue entry
 * @phba: Pointer to HBA context object.
 * @rcqe: Pointer to receive-queue completion queue entry.
 *
 * This routine process a receive-queue completion queue entry.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)
{
	bool workposted = false;
	struct fc_frame_header *fc_hdr;
	struct lpfc_queue *hrq = phba->sli4_hba.hdr_rq;
	struct lpfc_queue *drq = phba->sli4_hba.dat_rq;
	struct lpfc_nvmet_tgtport *tgtp;
	struct hbq_dmabuf *dma_buf;
	uint32_t status, rq_id;
	unsigned long iflags;

	/* sanity check on queue memory */
	if (unlikely(!hrq) || unlikely(!drq))
		return workposted;

	if (bf_get(lpfc_cqe_code, rcqe) == CQE_CODE_RECEIVE_V1)
		rq_id = bf_get(lpfc_rcqe_rq_id_v1, rcqe);
	else
		rq_id = bf_get(lpfc_rcqe_rq_id, rcqe);
	if (rq_id != hrq->queue_id)
		goto out;

	status = bf_get(lpfc_rcqe_status, rcqe);
	switch (status) {
	case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"2537 Receive Frame Truncated!!\n");
		/* fall through */
	case FC_STATUS_RQ_SUCCESS:
		spin_lock_irqsave(&phba->hbalock, iflags);
		lpfc_sli4_rq_release(hrq, drq);
		dma_buf = lpfc_sli_hbqbuf_get(&phba->hbqs[0].hbq_buffer_list);
		if (!dma_buf) {
			hrq->RQ_no_buf_found++;
			spin_unlock_irqrestore(&phba->hbalock, iflags);
			goto out;
		}
		hrq->RQ_rcv_buf++;
		hrq->RQ_buf_posted--;
		memcpy(&dma_buf->cq_event.cqe.rcqe_cmpl, rcqe, sizeof(*rcqe));

		fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;

		if (fc_hdr->fh_r_ctl == FC_RCTL_MDS_DIAGS ||
		    fc_hdr->fh_r_ctl == FC_RCTL_DD_UNSOL_DATA) {
			spin_unlock_irqrestore(&phba->hbalock, iflags);
			/* Handle MDS Loopback frames */
			lpfc_sli4_handle_mds_loopback(phba->pport, dma_buf);
			break;
		}

		/* save off the frame for the work thread to process */
		list_add_tail(&dma_buf->cq_event.list,
			      &phba->sli4_hba.sp_queue_event);
		/* Frame received */
		phba->hba_flag |= HBA_SP_QUEUE_EVT;
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		workposted = true;
		break;
	case FC_STATUS_INSUFF_BUF_FRM_DISC:
		if (phba->nvmet_support) {
			tgtp = phba->targetport->private;
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_NVME,
					"6402 RQE Error x%x, posted %d err_cnt "
					"%d: %x %x %x\n",
					status, hrq->RQ_buf_posted,
					hrq->RQ_no_posted_buf,
					atomic_read(&tgtp->rcv_fcp_cmd_in),
					atomic_read(&tgtp->rcv_fcp_cmd_out),
					atomic_read(&tgtp->xmt_fcp_release));
		}
		/* fallthrough */

	case FC_STATUS_INSUFF_BUF_NEED_BUF:
		hrq->RQ_no_posted_buf++;
		/* Post more buffers if possible */
		spin_lock_irqsave(&phba->hbalock, iflags);
		phba->hba_flag |= HBA_POST_RECEIVE_BUFFER;
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		workposted = true;
		break;
	}
out:
	return workposted;
}

/**
 * lpfc_sli4_sp_handle_cqe - Process a slow path completion queue entry
 * @phba: Pointer to HBA context object.
 * @cq: Pointer to the completion queue.
 * @cqe: Pointer to a completion queue entry.
 *
 * This routine process a slow-path work-queue or receive queue completion queue
 * entry.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			 struct lpfc_cqe *cqe)
{
	struct lpfc_cqe cqevt;
	bool workposted = false;

	/* Copy the work queue CQE and convert endian order if needed */
	lpfc_sli4_pcimem_bcopy(cqe, &cqevt, sizeof(struct lpfc_cqe));

	/* Check and process for different type of WCQE and dispatch */
	switch (bf_get(lpfc_cqe_code, &cqevt)) {
	case CQE_CODE_COMPL_WQE:
		/* Process the WQ/RQ complete event */
		phba->last_completion_time = jiffies;
		workposted = lpfc_sli4_sp_handle_els_wcqe(phba, cq,
				(struct lpfc_wcqe_complete *)&cqevt);
		break;
	case CQE_CODE_RELEASE_WQE:
		/* Process the WQ release event */
		lpfc_sli4_sp_handle_rel_wcqe(phba,
				(struct lpfc_wcqe_release *)&cqevt);
		break;
	case CQE_CODE_XRI_ABORTED:
		/* Process the WQ XRI abort event */
		phba->last_completion_time = jiffies;
		workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
				(struct sli4_wcqe_xri_aborted *)&cqevt);
		break;
	case CQE_CODE_RECEIVE:
	case CQE_CODE_RECEIVE_V1:
		/* Process the RQ event */
		phba->last_completion_time = jiffies;
		workposted = lpfc_sli4_sp_handle_rcqe(phba,
				(struct lpfc_rcqe *)&cqevt);
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0388 Not a valid WCQE code: x%x\n",
				bf_get(lpfc_cqe_code, &cqevt));
		break;
	}
	return workposted;
}

/**
 * lpfc_sli4_sp_handle_eqe - Process a slow-path event queue entry
 * @phba: Pointer to HBA context object.
 * @eqe: Pointer to fast-path event queue entry.
 *
 * This routine process a event queue entry from the slow-path event queue.
 * It will check the MajorCode and MinorCode to determine this is for a
 * completion event on a completion queue, if not, an error shall be logged
 * and just return. Otherwise, it will get to the corresponding completion
 * queue and process all the entries on that completion queue, rearm the
 * completion queue, and then return.
 *
 **/
static void
lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
	struct lpfc_queue *speq)
{
	struct lpfc_queue *cq = NULL, *childq;
	uint16_t cqid;

	/* Get the reference to the corresponding CQ */
	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);

	list_for_each_entry(childq, &speq->child_list, list) {
		if (childq->queue_id == cqid) {
			cq = childq;
			break;
		}
	}
	if (unlikely(!cq)) {
		if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"0365 Slow-path CQ identifier "
					"(%d) does not exist\n", cqid);
		return;
	}

	/* Save EQ associated with this CQ */
	cq->assoc_qp = speq;

	if (!queue_work_on(cq->chann, phba->wq, &cq->spwork))
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0390 Cannot schedule soft IRQ "
				"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
				cqid, cq->queue_id, raw_smp_processor_id());
}

/**
 * __lpfc_sli4_process_cq - Process elements of a CQ
 * @phba: Pointer to HBA context object.
 * @cq: Pointer to CQ to be processed
 * @handler: Routine to process each cqe
 * @delay: Pointer to usdelay to set in case of rescheduling of the handler
 *
 * This routine processes completion queue entries in a CQ. While a valid
 * queue element is found, the handler is called. During processing checks
 * are made for periodic doorbell writes to let the hardware know of
 * element consumption.
 *
 * If the max limit on cqes to process is hit, or there are no more valid
 * entries, the loop stops. If we processed a sufficient number of elements,
 * meaning there is sufficient load, rather than rearming and generating
 * another interrupt, a cq rescheduling delay will be set. A delay of 0
 * indicates no rescheduling.
 *
 * Returns True if work scheduled, False otherwise.
 **/
static bool
__lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq,
	bool (*handler)(struct lpfc_hba *, struct lpfc_queue *,
			struct lpfc_cqe *), unsigned long *delay)
{
	struct lpfc_cqe *cqe;
	bool workposted = false;
	int count = 0, consumed = 0;
	bool arm = true;

	/* default - no reschedule */
	*delay = 0;

	if (cmpxchg(&cq->queue_claimed, 0, 1) != 0)
		goto rearm_and_exit;

	/* Process all the entries to the CQ */
	cqe = lpfc_sli4_cq_get(cq);
	while (cqe) {
#if defined(CONFIG_SCSI_LPFC_DEBUG_FS) && defined(BUILD_NVME)
		if (phba->ktime_on)
			cq->isr_timestamp = ktime_get_ns();
		else
			cq->isr_timestamp = 0;
#endif
		workposted |= handler(phba, cq, cqe);
		__lpfc_sli4_consume_cqe(phba, cq, cqe);

		consumed++;
		if (!(++count % cq->max_proc_limit))
			break;

		if (!(count % cq->notify_interval)) {
			phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed,
						LPFC_QUEUE_NOARM);
			consumed = 0;
		}

		cqe = lpfc_sli4_cq_get(cq);
	}
	if (count >= phba->cfg_cq_poll_threshold) {
		*delay = 1;
		arm = false;
	}

	/* Track the max number of CQEs processed in 1 EQ */
	if (count > cq->CQ_max_cqe)
		cq->CQ_max_cqe = count;

	cq->assoc_qp->EQ_cqe_cnt += count;

	/* Catch the no cq entry condition */
	if (unlikely(count == 0))
		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"0369 No entry from completion queue "
				"qid=%d\n", cq->queue_id);

	cq->queue_claimed = 0;

rearm_and_exit:
	phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed,
			arm ?  LPFC_QUEUE_REARM : LPFC_QUEUE_NOARM);

	return workposted;
}

/**
 * lpfc_sli4_sp_process_cq - Process a slow-path event queue entry
 * @cq: pointer to CQ to process
 *
 * This routine calls the cq processing routine with a handler specific
 * to the type of queue bound to it.
 *
 * The CQ routine returns two values: the first is the calling status,
 * which indicates whether work was queued to the  background discovery
 * thread. If true, the routine should wakeup the discovery thread;
 * the second is the delay parameter. If non-zero, rather than rearming
 * the CQ and yet another interrupt, the CQ handler should be queued so
 * that it is processed in a subsequent polling action. The value of
 * the delay indicates when to reschedule it.
 **/
static void
__lpfc_sli4_sp_process_cq(struct lpfc_queue *cq)
{
	struct lpfc_hba *phba = cq->phba;
	unsigned long delay;
	bool workposted = false;

	/* Process and rearm the CQ */
	switch (cq->type) {
	case LPFC_MCQ:
		workposted |= __lpfc_sli4_process_cq(phba, cq,
						lpfc_sli4_sp_handle_mcqe,
						&delay);
		break;
	case LPFC_WCQ:
		if (cq->subtype == LPFC_FCP || cq->subtype == LPFC_NVME)
			workposted |= __lpfc_sli4_process_cq(phba, cq,
						lpfc_sli4_fp_handle_cqe,
						&delay);
		else
			workposted |= __lpfc_sli4_process_cq(phba, cq,
						lpfc_sli4_sp_handle_cqe,
						&delay);
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0370 Invalid completion queue type (%d)\n",
				cq->type);
		return;
	}

	if (delay) {
		if (!queue_delayed_work_on(cq->chann, phba->wq,
					   &cq->sched_spwork, delay))
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0394 Cannot schedule soft IRQ "
				"for cqid=%d on CPU %d\n",
				cq->queue_id, cq->chann);
	}

	/* wake up worker thread if there are works to be done */
	if (workposted)
		lpfc_worker_wake_up(phba);
}

/**
 * lpfc_sli4_sp_process_cq - slow-path work handler when started by
 *   interrupt
 * @work: pointer to work element
 *
 * translates from the work handler and calls the slow-path handler.
 **/
static void
lpfc_sli4_sp_process_cq(struct work_struct *work)
{
	struct lpfc_queue *cq = container_of(work, struct lpfc_queue, spwork);

	__lpfc_sli4_sp_process_cq(cq);
}

/**
 * lpfc_sli4_dly_sp_process_cq - slow-path work handler when started by timer
 * @work: pointer to work element
 *
 * translates from the work handler and calls the slow-path handler.
 **/
static void
lpfc_sli4_dly_sp_process_cq(struct work_struct *work)
{
	struct lpfc_queue *cq = container_of(to_delayed_work(work),
					struct lpfc_queue, sched_spwork);

	__lpfc_sli4_sp_process_cq(cq);
}

/**
 * lpfc_sli4_fp_handle_fcp_wcqe - Process fast-path work queue completion entry
 * @phba: Pointer to HBA context object.
 * @cq: Pointer to associated CQ
 * @wcqe: Pointer to work-queue completion queue entry.
 *
 * This routine process a fast-path work queue completion entry from fast-path
 * event queue for FCP command response completion.
 **/
static void
lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			     struct lpfc_wcqe_complete *wcqe)
{
	struct lpfc_sli_ring *pring = cq->pring;
	struct lpfc_iocbq *cmdiocbq;
	struct lpfc_iocbq irspiocbq;
	unsigned long iflags;

	/* Check for response status */
	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
		/* If resource errors reported from HBA, reduce queue
		 * depth of the SCSI device.
		 */
		if (((bf_get(lpfc_wcqe_c_status, wcqe) ==
		     IOSTAT_LOCAL_REJECT)) &&
		    ((wcqe->parameter & IOERR_PARAM_MASK) ==
		     IOERR_NO_RESOURCES))
			phba->lpfc_rampdown_queue_depth(phba);

		/* Log the error status */
		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
				"0373 FCP CQE error: status=x%x: "
				"CQE: %08x %08x %08x %08x\n",
				bf_get(lpfc_wcqe_c_status, wcqe),
				wcqe->word0, wcqe->total_data_placed,
				wcqe->parameter, wcqe->word3);
	}

	/* Look up the FCP command IOCB and create pseudo response IOCB */
	spin_lock_irqsave(&pring->ring_lock, iflags);
	pring->stats.iocb_event++;
	spin_unlock_irqrestore(&pring->ring_lock, iflags);
	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
				bf_get(lpfc_wcqe_c_request_tag, wcqe));
	if (unlikely(!cmdiocbq)) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"0374 FCP complete with no corresponding "
				"cmdiocb: iotag (%d)\n",
				bf_get(lpfc_wcqe_c_request_tag, wcqe));
		return;
	}
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
	cmdiocbq->isr_timestamp = cq->isr_timestamp;
#endif
	if (cmdiocbq->iocb_cmpl == NULL) {
		if (cmdiocbq->wqe_cmpl) {
			if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) {
				spin_lock_irqsave(&phba->hbalock, iflags);
				cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED;
				spin_unlock_irqrestore(&phba->hbalock, iflags);
			}

			/* Pass the cmd_iocb and the wcqe to the upper layer */
			(cmdiocbq->wqe_cmpl)(phba, cmdiocbq, wcqe);
			return;
		}
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"0375 FCP cmdiocb not callback function "
				"iotag: (%d)\n",
				bf_get(lpfc_wcqe_c_request_tag, wcqe));
		return;
	}

	/* Fake the irspiocb and copy necessary response information */
	lpfc_sli4_iocb_param_transfer(phba, &irspiocbq, cmdiocbq, wcqe);

	if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) {
		spin_lock_irqsave(&phba->hbalock, iflags);
		cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED;
		spin_unlock_irqrestore(&phba->hbalock, iflags);
	}

	/* Pass the cmd_iocb and the rsp state to the upper layer */
	(cmdiocbq->iocb_cmpl)(phba, cmdiocbq, &irspiocbq);
}

/**
 * lpfc_sli4_fp_handle_rel_wcqe - Handle fast-path WQ entry consumed event
 * @phba: Pointer to HBA context object.
 * @cq: Pointer to completion queue.
 * @wcqe: Pointer to work-queue completion queue entry.
 *
 * This routine handles an fast-path WQ entry consumed event by invoking the
 * proper WQ release routine to the slow-path WQ.
 **/
static void
lpfc_sli4_fp_handle_rel_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			     struct lpfc_wcqe_release *wcqe)
{
	struct lpfc_queue *childwq;
	bool wqid_matched = false;
	uint16_t hba_wqid;

	/* Check for fast-path FCP work queue release */
	hba_wqid = bf_get(lpfc_wcqe_r_wq_id, wcqe);
	list_for_each_entry(childwq, &cq->child_list, list) {
		if (childwq->queue_id == hba_wqid) {
			lpfc_sli4_wq_release(childwq,
					bf_get(lpfc_wcqe_r_wqe_index, wcqe));
			if (childwq->q_flag & HBA_NVMET_WQFULL)
				lpfc_nvmet_wqfull_process(phba, childwq);
			wqid_matched = true;
			break;
		}
	}
	/* Report warning log message if no match found */
	if (wqid_matched != true)
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"2580 Fast-path wqe consume event carries "
				"miss-matched qid: wcqe-qid=x%x\n", hba_wqid);
}

/**
 * lpfc_sli4_nvmet_handle_rcqe - Process a receive-queue completion queue entry
 * @phba: Pointer to HBA context object.
 * @rcqe: Pointer to receive-queue completion queue entry.
 *
 * This routine process a receive-queue completion queue entry.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			    struct lpfc_rcqe *rcqe)
{
	bool workposted = false;
	struct lpfc_queue *hrq;
	struct lpfc_queue *drq;
	struct rqb_dmabuf *dma_buf;
	struct fc_frame_header *fc_hdr;
	struct lpfc_nvmet_tgtport *tgtp;
	uint32_t status, rq_id;
	unsigned long iflags;
	uint32_t fctl, idx;

	if ((phba->nvmet_support == 0) ||
	    (phba->sli4_hba.nvmet_cqset == NULL))
		return workposted;

	idx = cq->queue_id - phba->sli4_hba.nvmet_cqset[0]->queue_id;
	hrq = phba->sli4_hba.nvmet_mrq_hdr[idx];
	drq = phba->sli4_hba.nvmet_mrq_data[idx];

	/* sanity check on queue memory */
	if (unlikely(!hrq) || unlikely(!drq))
		return workposted;

	if (bf_get(lpfc_cqe_code, rcqe) == CQE_CODE_RECEIVE_V1)
		rq_id = bf_get(lpfc_rcqe_rq_id_v1, rcqe);
	else
		rq_id = bf_get(lpfc_rcqe_rq_id, rcqe);

	if ((phba->nvmet_support == 0) ||
	    (rq_id != hrq->queue_id))
		return workposted;

	status = bf_get(lpfc_rcqe_status, rcqe);
	switch (status) {
	case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"6126 Receive Frame Truncated!!\n");
		/* fall through */
	case FC_STATUS_RQ_SUCCESS:
		spin_lock_irqsave(&phba->hbalock, iflags);
		lpfc_sli4_rq_release(hrq, drq);
		dma_buf = lpfc_sli_rqbuf_get(phba, hrq);
		if (!dma_buf) {
			hrq->RQ_no_buf_found++;
			spin_unlock_irqrestore(&phba->hbalock, iflags);
			goto out;
		}
		spin_unlock_irqrestore(&phba->hbalock, iflags);
		hrq->RQ_rcv_buf++;
		hrq->RQ_buf_posted--;
		fc_hdr = (struct fc_frame_header *)dma_buf->hbuf.virt;

		/* Just some basic sanity checks on FCP Command frame */
		fctl = (fc_hdr->fh_f_ctl[0] << 16 |
		fc_hdr->fh_f_ctl[1] << 8 |
		fc_hdr->fh_f_ctl[2]);
		if (((fctl &
		    (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) !=
		    (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) ||
		    (fc_hdr->fh_seq_cnt != 0)) /* 0 byte swapped is still 0 */
			goto drop;

		if (fc_hdr->fh_type == FC_TYPE_FCP) {
			dma_buf->bytes_recv = bf_get(lpfc_rcqe_length,  rcqe);
			lpfc_nvmet_unsol_fcp_event(
				phba, idx, dma_buf,
				cq->isr_timestamp);
			return false;
		}
drop:
		lpfc_rq_buf_free(phba, &dma_buf->hbuf);
		break;
	case FC_STATUS_INSUFF_BUF_FRM_DISC:
		if (phba->nvmet_support) {
			tgtp = phba->targetport->private;
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_NVME,
					"6401 RQE Error x%x, posted %d err_cnt "
					"%d: %x %x %x\n",
					status, hrq->RQ_buf_posted,
					hrq->RQ_no_posted_buf,
					atomic_read(&tgtp->rcv_fcp_cmd_in),
					atomic_read(&tgtp->rcv_fcp_cmd_out),
					atomic_read(&tgtp->xmt_fcp_release));
		}
		/* fallthrough */

	case FC_STATUS_INSUFF_BUF_NEED_BUF:
		hrq->RQ_no_posted_buf++;
		/* Post more buffers if possible */
		break;
	}
out:
	return workposted;
}

/**
 * lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry
 * @phba: adapter with cq
 * @cq: Pointer to the completion queue.
 * @eqe: Pointer to fast-path completion queue entry.
 *
 * This routine process a fast-path work queue completion entry from fast-path
 * event queue for FCP command response completion.
 *
 * Return: true if work posted to worker thread, otherwise false.
 **/
static bool
lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
			 struct lpfc_cqe *cqe)
{
	struct lpfc_wcqe_release wcqe;
	bool workposted = false;

	/* Copy the work queue CQE and convert endian order if needed */
	lpfc_sli4_pcimem_bcopy(cqe, &wcqe, sizeof(struct lpfc_cqe));

	/* Check and process for different type of WCQE and dispatch */
	switch (bf_get(lpfc_wcqe_c_code, &wcqe)) {
	case CQE_CODE_COMPL_WQE:
	case CQE_CODE_NVME_ERSP:
		cq->CQ_wq++;
		/* Process the WQ complete event */
		phba->last_completion_time = jiffies;
		if ((cq->subtype == LPFC_FCP) || (cq->subtype == LPFC_NVME))
			lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
				(struct lpfc_wcqe_complete *)&wcqe);
		if (cq->subtype == LPFC_NVME_LS)
			lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
				(struct lpfc_wcqe_complete *)&wcqe);
		break;
	case CQE_CODE_RELEASE_WQE:
		cq->CQ_release_wqe++;
		/* Process the WQ release event */
		lpfc_sli4_fp_handle_rel_wcqe(phba, cq,
				(struct lpfc_wcqe_release *)&wcqe);
		break;
	case CQE_CODE_XRI_ABORTED:
		cq->CQ_xri_aborted++;
		/* Process the WQ XRI abort event */
		phba->last_completion_time = jiffies;
		workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
				(struct sli4_wcqe_xri_aborted *)&wcqe);
		break;
	case CQE_CODE_RECEIVE_V1:
	case CQE_CODE_RECEIVE:
		phba->last_completion_time = jiffies;
		if (cq->subtype == LPFC_NVMET) {
			workposted = lpfc_sli4_nvmet_handle_rcqe(
				phba, cq, (struct lpfc_rcqe *)&wcqe);
		}
		break;
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0144 Not a valid CQE code: x%x\n",
				bf_get(lpfc_wcqe_c_code, &wcqe));
		break;
	}
	return workposted;
}

/**
 * lpfc_sli4_hba_handle_eqe - Process a fast-path event queue entry
 * @phba: Pointer to HBA context object.
 * @eqe: Pointer to fast-path event queue entry.
 *
 * This routine process a event queue entry from the fast-path event queue.
 * It will check the MajorCode and MinorCode to determine this is for a
 * completion event on a completion queue, if not, an error shall be logged
 * and just return. Otherwise, it will get to the corresponding completion
 * queue and process all the entries on the completion queue, rearm the
 * completion queue, and then return.
 **/
static void
lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq,
			 struct lpfc_eqe *eqe)
{
	struct lpfc_queue *cq = NULL;
	uint32_t qidx = eq->hdwq;
	uint16_t cqid, id;

	if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0366 Not a valid completion "
				"event: majorcode=x%x, minorcode=x%x\n",
				bf_get_le32(lpfc_eqe_major_code, eqe),
				bf_get_le32(lpfc_eqe_minor_code, eqe));
		return;
	}

	/* Get the reference to the corresponding CQ */
	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);

	/* Use the fast lookup method first */
	if (cqid <= phba->sli4_hba.cq_max) {
		cq = phba->sli4_hba.cq_lookup[cqid];
		if (cq)
			goto  work_cq;
	}

	/* Next check for NVMET completion */
	if (phba->cfg_nvmet_mrq && phba->sli4_hba.nvmet_cqset) {
		id = phba->sli4_hba.nvmet_cqset[0]->queue_id;
		if ((cqid >= id) && (cqid < (id + phba->cfg_nvmet_mrq))) {
			/* Process NVMET unsol rcv */
			cq = phba->sli4_hba.nvmet_cqset[cqid - id];
			goto  process_cq;
		}
	}

	if (phba->sli4_hba.nvmels_cq &&
	    (cqid == phba->sli4_hba.nvmels_cq->queue_id)) {
		/* Process NVME unsol rcv */
		cq = phba->sli4_hba.nvmels_cq;
	}

	/* Otherwise this is a Slow path event */
	if (cq == NULL) {
		lpfc_sli4_sp_handle_eqe(phba, eqe,
					phba->sli4_hba.hdwq[qidx].hba_eq);
		return;
	}

process_cq:
	if (unlikely(cqid != cq->queue_id)) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0368 Miss-matched fast-path completion "
				"queue identifier: eqcqid=%d, fcpcqid=%d\n",
				cqid, cq->queue_id);
		return;
	}

work_cq:
	if (!queue_work_on(cq->chann, phba->wq, &cq->irqwork))
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0363 Cannot schedule soft IRQ "
				"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
				cqid, cq->queue_id, raw_smp_processor_id());
}

/**
 * __lpfc_sli4_hba_process_cq - Process a fast-path event queue entry
 * @cq: Pointer to CQ to be processed
 *
 * This routine calls the cq processing routine with the handler for
 * fast path CQEs.
 *
 * The CQ routine returns two values: the first is the calling status,
 * which indicates whether work was queued to the  background discovery
 * thread. If true, the routine should wakeup the discovery thread;
 * the second is the delay parameter. If non-zero, rather than rearming
 * the CQ and yet another interrupt, the CQ handler should be queued so
 * that it is processed in a subsequent polling action. The value of
 * the delay indicates when to reschedule it.
 **/
static void
__lpfc_sli4_hba_process_cq(struct lpfc_queue *cq)
{
	struct lpfc_hba *phba = cq->phba;
	unsigned long delay;
	bool workposted = false;

	/* process and rearm the CQ */
	workposted |= __lpfc_sli4_process_cq(phba, cq, lpfc_sli4_fp_handle_cqe,
					     &delay);

	if (delay) {
		if (!queue_delayed_work_on(cq->chann, phba->wq,
					   &cq->sched_irqwork, delay))
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0367 Cannot schedule soft IRQ "
				"for cqid=%d on CPU %d\n",
				cq->queue_id, cq->chann);
	}

	/* wake up worker thread if there are works to be done */
	if (workposted)
		lpfc_worker_wake_up(phba);
}

/**
 * lpfc_sli4_hba_process_cq - fast-path work handler when started by
 *   interrupt
 * @work: pointer to work element
 *
 * translates from the work handler and calls the fast-path handler.
 **/
static void
lpfc_sli4_hba_process_cq(struct work_struct *work)
{
	struct lpfc_queue *cq = container_of(work, struct lpfc_queue, irqwork);

	__lpfc_sli4_hba_process_cq(cq);
}

/**
 * lpfc_sli4_hba_process_cq - fast-path work handler when started by timer
 * @work: pointer to work element
 *
 * translates from the work handler and calls the fast-path handler.
 **/
static void
lpfc_sli4_dly_hba_process_cq(struct work_struct *work)
{
	struct lpfc_queue *cq = container_of(to_delayed_work(work),
					struct lpfc_queue, sched_irqwork);

	__lpfc_sli4_hba_process_cq(cq);
}

/**
 * lpfc_sli4_hba_intr_handler - HBA interrupt handler to SLI-4 device
 * @irq: Interrupt number.
 * @dev_id: The device context pointer.
 *
 * This function is directly called from the PCI layer as an interrupt
 * service routine when device with SLI-4 interface spec is enabled with
 * MSI-X multi-message interrupt mode and there is a fast-path FCP IOCB
 * ring event in the HBA. However, when the device is enabled with either
 * MSI or Pin-IRQ interrupt mode, this function is called as part of the
 * device-level interrupt handler. When the PCI slot is in error recovery
 * or the HBA is undergoing initialization, the interrupt handler will not
 * process the interrupt. The SCSI FCP fast-path ring event are handled in
 * the intrrupt context. This function is called without any lock held.
 * It gets the hbalock to access and update SLI data structures. Note that,
 * the FCP EQ to FCP CQ are one-to-one map such that the FCP EQ index is
 * equal to that of FCP CQ index.
 *
 * The link attention and ELS ring attention events are handled
 * by the worker thread. The interrupt handler signals the worker thread
 * and returns for these events. This function is called without any lock
 * held. It gets the hbalock to access and update SLI data structures.
 *
 * This function returns IRQ_HANDLED when interrupt is handled else it
 * returns IRQ_NONE.
 **/
irqreturn_t
lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
{
	struct lpfc_hba *phba;
	struct lpfc_hba_eq_hdl *hba_eq_hdl;
	struct lpfc_queue *fpeq;
	unsigned long iflag;
	int ecount = 0;
	int hba_eqidx;
	struct lpfc_eq_intr_info *eqi;
	uint32_t icnt;

	/* Get the driver's phba structure from the dev_id */
	hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id;
	phba = hba_eq_hdl->phba;
	hba_eqidx = hba_eq_hdl->idx;

	if (unlikely(!phba))
		return IRQ_NONE;
	if (unlikely(!phba->sli4_hba.hdwq))
		return IRQ_NONE;

	/* Get to the EQ struct associated with this vector */
	fpeq = phba->sli4_hba.hdwq[hba_eqidx].hba_eq;
	if (unlikely(!fpeq))
		return IRQ_NONE;

	/* Check device state for handling interrupt */
	if (unlikely(lpfc_intr_state_check(phba))) {
		/* Check again for link_state with lock held */
		spin_lock_irqsave(&phba->hbalock, iflag);
		if (phba->link_state < LPFC_LINK_DOWN)
			/* Flush, clear interrupt, and rearm the EQ */
			lpfc_sli4_eq_flush(phba, fpeq);
		spin_unlock_irqrestore(&phba->hbalock, iflag);
		return IRQ_NONE;
	}

	eqi = phba->sli4_hba.eq_info;
	icnt = this_cpu_inc_return(eqi->icnt);
	fpeq->last_cpu = raw_smp_processor_id();

	if (icnt > LPFC_EQD_ISR_TRIGGER &&
	    phba->cfg_irq_chann == 1 &&
	    phba->cfg_auto_imax &&
	    fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY &&
	    phba->sli.sli_flag & LPFC_SLI_USE_EQDR)
		lpfc_sli4_mod_hba_eq_delay(phba, fpeq, LPFC_MAX_AUTO_EQ_DELAY);

	/* process and rearm the EQ */
	ecount = lpfc_sli4_process_eq(phba, fpeq);

	if (unlikely(ecount == 0)) {
		fpeq->EQ_no_entry++;
		if (phba->intr_type == MSIX)
			/* MSI-X treated interrupt served as no EQ share INT */
			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
					"0358 MSI-X interrupt with no EQE\n");
		else
			/* Non MSI-X treated on interrupt as EQ share INT */
			return IRQ_NONE;
	}

	return IRQ_HANDLED;
} /* lpfc_sli4_fp_intr_handler */

/**
 * lpfc_sli4_intr_handler - Device-level interrupt handler for SLI-4 device
 * @irq: Interrupt number.
 * @dev_id: The device context pointer.
 *
 * This function is the device-level interrupt handler to device with SLI-4
 * interface spec, called from the PCI layer when either MSI or Pin-IRQ
 * interrupt mode is enabled and there is an event in the HBA which requires
 * driver attention. This function invokes the slow-path interrupt attention
 * handling function and fast-path interrupt attention handling function in
 * turn to process the relevant HBA attention events. This function is called
 * without any lock held. It gets the hbalock to access and update SLI data
 * structures.
 *
 * This function returns IRQ_HANDLED when interrupt is handled, else it
 * returns IRQ_NONE.
 **/
irqreturn_t
lpfc_sli4_intr_handler(int irq, void *dev_id)
{
	struct lpfc_hba  *phba;
	irqreturn_t hba_irq_rc;
	bool hba_handled = false;
	int qidx;

	/* Get the driver's phba structure from the dev_id */
	phba = (struct lpfc_hba *)dev_id;

	if (unlikely(!phba))
		return IRQ_NONE;

	/*
	 * Invoke fast-path host attention interrupt handling as appropriate.
	 */
	for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) {
		hba_irq_rc = lpfc_sli4_hba_intr_handler(irq,
					&phba->sli4_hba.hba_eq_hdl[qidx]);
		if (hba_irq_rc == IRQ_HANDLED)
			hba_handled |= true;
	}

	return (hba_handled == true) ? IRQ_HANDLED : IRQ_NONE;
} /* lpfc_sli4_intr_handler */

/**
 * lpfc_sli4_queue_free - free a queue structure and associated memory
 * @queue: The queue structure to free.
 *
 * This function frees a queue structure and the DMAable memory used for
 * the host resident queue. This function must be called after destroying the
 * queue on the HBA.
 **/
void
lpfc_sli4_queue_free(struct lpfc_queue *queue)
{
	struct lpfc_dmabuf *dmabuf;

	if (!queue)
		return;

	if (!list_empty(&queue->wq_list))
		list_del(&queue->wq_list);

	while (!list_empty(&queue->page_list)) {
		list_remove_head(&queue->page_list, dmabuf, struct lpfc_dmabuf,
				 list);
		dma_free_coherent(&queue->phba->pcidev->dev, queue->page_size,
				  dmabuf->virt, dmabuf->phys);
		kfree(dmabuf);
	}
	if (queue->rqbp) {
		lpfc_free_rq_buffer(queue->phba, queue);
		kfree(queue->rqbp);
	}

	if (!list_empty(&queue->cpu_list))
		list_del(&queue->cpu_list);

	kfree(queue);
	return;
}

/**
 * lpfc_sli4_queue_alloc - Allocate and initialize a queue structure
 * @phba: The HBA that this queue is being created on.
 * @page_size: The size of a queue page
 * @entry_size: The size of each queue entry for this queue.
 * @entry count: The number of entries that this queue will handle.
 * @cpu: The cpu that will primarily utilize this queue.
 *
 * This function allocates a queue structure and the DMAable memory used for
 * the host resident queue. This function must be called before creating the
 * queue on the HBA.
 **/
struct lpfc_queue *
lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size,
		      uint32_t entry_size, uint32_t entry_count, int cpu)
{
	struct lpfc_queue *queue;
	struct lpfc_dmabuf *dmabuf;
	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
	uint16_t x, pgcnt;

	if (!phba->sli4_hba.pc_sli4_params.supported)
		hw_page_size = page_size;

	pgcnt = ALIGN(entry_size * entry_count, hw_page_size) / hw_page_size;

	/* If needed, Adjust page count to match the max the adapter supports */
	if (pgcnt > phba->sli4_hba.pc_sli4_params.wqpcnt)
		pgcnt = phba->sli4_hba.pc_sli4_params.wqpcnt;

	queue = kzalloc_node(sizeof(*queue) + (sizeof(void *) * pgcnt),
			     GFP_KERNEL, cpu_to_node(cpu));
	if (!queue)
		return NULL;

	INIT_LIST_HEAD(&queue->list);
	INIT_LIST_HEAD(&queue->wq_list);
	INIT_LIST_HEAD(&queue->wqfull_list);
	INIT_LIST_HEAD(&queue->page_list);
	INIT_LIST_HEAD(&queue->child_list);
	INIT_LIST_HEAD(&queue->cpu_list);

	/* Set queue parameters now.  If the system cannot provide memory
	 * resources, the free routine needs to know what was allocated.
	 */
	queue->page_count = pgcnt;
	queue->q_pgs = (void **)&queue[1];
	queue->entry_cnt_per_pg = hw_page_size / entry_size;
	queue->entry_size = entry_size;
	queue->entry_count = entry_count;
	queue->page_size = hw_page_size;
	queue->phba = phba;

	for (x = 0; x < queue->page_count; x++) {
		dmabuf = kzalloc_node(sizeof(*dmabuf), GFP_KERNEL,
				      dev_to_node(&phba->pcidev->dev));
		if (!dmabuf)
			goto out_fail;
		dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev,
						  hw_page_size, &dmabuf->phys,
						  GFP_KERNEL);
		if (!dmabuf->virt) {
			kfree(dmabuf);
			goto out_fail;
		}
		dmabuf->buffer_tag = x;
		list_add_tail(&dmabuf->list, &queue->page_list);
		/* use lpfc_sli4_qe to index a paritcular entry in this page */
		queue->q_pgs[x] = dmabuf->virt;
	}
	INIT_WORK(&queue->irqwork, lpfc_sli4_hba_process_cq);
	INIT_WORK(&queue->spwork, lpfc_sli4_sp_process_cq);
	INIT_DELAYED_WORK(&queue->sched_irqwork, lpfc_sli4_dly_hba_process_cq);
	INIT_DELAYED_WORK(&queue->sched_spwork, lpfc_sli4_dly_sp_process_cq);

	/* notify_interval will be set during q creation */

	return queue;
out_fail:
	lpfc_sli4_queue_free(queue);
	return NULL;
}

/**
 * lpfc_dual_chute_pci_bar_map - Map pci base address register to host memory
 * @phba: HBA structure that indicates port to create a queue on.
 * @pci_barset: PCI BAR set flag.
 *
 * This function shall perform iomap of the specified PCI BAR address to host
 * memory address if not already done so and return it. The returned host
 * memory address can be NULL.
 */
static void __iomem *
lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset)
{
	if (!phba->pcidev)
		return NULL;

	switch (pci_barset) {
	case WQ_PCI_BAR_0_AND_1:
		return phba->pci_bar0_memmap_p;
	case WQ_PCI_BAR_2_AND_3:
		return phba->pci_bar2_memmap_p;
	case WQ_PCI_BAR_4_AND_5:
		return phba->pci_bar4_memmap_p;
	default:
		break;
	}
	return NULL;
}

/**
 * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs
 * @phba: HBA structure that EQs are on.
 * @startq: The starting EQ index to modify
 * @numq: The number of EQs (consecutive indexes) to modify
 * @usdelay: amount of delay
 *
 * This function revises the EQ delay on 1 or more EQs. The EQ delay
 * is set either by writing to a register (if supported by the SLI Port)
 * or by mailbox command. The mailbox command allows several EQs to be
 * updated at once.
 *
 * The @phba struct is used to send a mailbox command to HBA. The @startq
 * is used to get the starting EQ index to change. The @numq value is
 * used to specify how many consecutive EQ indexes, starting at EQ index,
 * are to be changed. This function is asynchronous and will wait for any
 * mailbox commands to finish before returning.
 *
 * On success this function will return a zero. If unable to allocate
 * enough memory this function will return -ENOMEM. If a mailbox command
 * fails this function will return -ENXIO. Note: on ENXIO, some EQs may
 * have had their delay multipler changed.
 **/
void
lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq,
			 uint32_t numq, uint32_t usdelay)
{
	struct lpfc_mbx_modify_eq_delay *eq_delay;
	LPFC_MBOXQ_t *mbox;
	struct lpfc_queue *eq;
	int cnt = 0, rc, length;
	uint32_t shdr_status, shdr_add_status;
	uint32_t dmult;
	int qidx;
	union lpfc_sli4_cfg_shdr *shdr;

	if (startq >= phba->cfg_irq_chann)
		return;

	if (usdelay > 0xFFFF) {
		lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_FCP | LOG_NVME,
				"6429 usdelay %d too large. Scaled down to "
				"0xFFFF.\n", usdelay);
		usdelay = 0xFFFF;
	}

	/* set values by EQ_DELAY register if supported */
	if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) {
		for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) {
			eq = phba->sli4_hba.hdwq[qidx].hba_eq;
			if (!eq)
				continue;

			lpfc_sli4_mod_hba_eq_delay(phba, eq, usdelay);

			if (++cnt >= numq)
				break;
		}

		return;
	}

	/* Otherwise, set values by mailbox cmd */

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_FCP | LOG_NVME,
				"6428 Failed allocating mailbox cmd buffer."
				" EQ delay was not set.\n");
		return;
	}
	length = (sizeof(struct lpfc_mbx_modify_eq_delay) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_MODIFY_EQ_DELAY,
			 length, LPFC_SLI4_MBX_EMBED);
	eq_delay = &mbox->u.mqe.un.eq_delay;

	/* Calculate delay multiper from maximum interrupt per second */
	dmult = (usdelay * LPFC_DMULT_CONST) / LPFC_SEC_TO_USEC;
	if (dmult)
		dmult--;
	if (dmult > LPFC_DMULT_MAX)
		dmult = LPFC_DMULT_MAX;

	for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) {
		eq = phba->sli4_hba.hdwq[qidx].hba_eq;
		if (!eq)
			continue;
		eq->q_mode = usdelay;
		eq_delay->u.request.eq[cnt].eq_id = eq->queue_id;
		eq_delay->u.request.eq[cnt].phase = 0;
		eq_delay->u.request.eq[cnt].delay_multi = dmult;

		if (++cnt >= numq)
			break;
	}
	eq_delay->u.request.num_eq = cnt;

	mbox->vport = phba->pport;
	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
	mbox->ctx_buf = NULL;
	mbox->ctx_ndlp = NULL;
	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	shdr = (union lpfc_sli4_cfg_shdr *) &eq_delay->header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2512 MODIFY_EQ_DELAY mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
	}
	mempool_free(mbox, phba->mbox_mem_pool);
	return;
}

/**
 * lpfc_eq_create - Create an Event Queue on the HBA
 * @phba: HBA structure that indicates port to create a queue on.
 * @eq: The queue structure to use to create the event queue.
 * @imax: The maximum interrupt per second limit.
 *
 * This function creates an event queue, as detailed in @eq, on a port,
 * described by @phba by sending an EQ_CREATE mailbox command to the HBA.
 *
 * The @phba struct is used to send mailbox command to HBA. The @eq struct
 * is used to get the entry count and entry size that are necessary to
 * determine the number of pages to allocate and use for this queue. This
 * function will send the EQ_CREATE mailbox command to the HBA to setup the
 * event queue. This function is asynchronous and will wait for the mailbox
 * command to finish before continuing.
 *
 * On success this function will return a zero. If unable to allocate enough
 * memory this function will return -ENOMEM. If the queue create mailbox command
 * fails this function will return -ENXIO.
 **/
int
lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax)
{
	struct lpfc_mbx_eq_create *eq_create;
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	struct lpfc_dmabuf *dmabuf;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	uint16_t dmult;
	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;

	/* sanity check on queue memory */
	if (!eq)
		return -ENODEV;
	if (!phba->sli4_hba.pc_sli4_params.supported)
		hw_page_size = SLI4_PAGE_SIZE;

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_eq_create) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_EQ_CREATE,
			 length, LPFC_SLI4_MBX_EMBED);
	eq_create = &mbox->u.mqe.un.eq_create;
	shdr = (union lpfc_sli4_cfg_shdr *) &eq_create->header.cfg_shdr;
	bf_set(lpfc_mbx_eq_create_num_pages, &eq_create->u.request,
	       eq->page_count);
	bf_set(lpfc_eq_context_size, &eq_create->u.request.context,
	       LPFC_EQE_SIZE);
	bf_set(lpfc_eq_context_valid, &eq_create->u.request.context, 1);

	/* Use version 2 of CREATE_EQ if eqav is set */
	if (phba->sli4_hba.pc_sli4_params.eqav) {
		bf_set(lpfc_mbox_hdr_version, &shdr->request,
		       LPFC_Q_CREATE_VERSION_2);
		bf_set(lpfc_eq_context_autovalid, &eq_create->u.request.context,
		       phba->sli4_hba.pc_sli4_params.eqav);
	}

	/* don't setup delay multiplier using EQ_CREATE */
	dmult = 0;
	bf_set(lpfc_eq_context_delay_multi, &eq_create->u.request.context,
	       dmult);
	switch (eq->entry_count) {
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0360 Unsupported EQ count. (%d)\n",
				eq->entry_count);
		if (eq->entry_count < 256)
			return -EINVAL;
		/* fall through - otherwise default to smallest count */
	case 256:
		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
		       LPFC_EQ_CNT_256);
		break;
	case 512:
		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
		       LPFC_EQ_CNT_512);
		break;
	case 1024:
		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
		       LPFC_EQ_CNT_1024);
		break;
	case 2048:
		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
		       LPFC_EQ_CNT_2048);
		break;
	case 4096:
		bf_set(lpfc_eq_context_count, &eq_create->u.request.context,
		       LPFC_EQ_CNT_4096);
		break;
	}
	list_for_each_entry(dmabuf, &eq->page_list, list) {
		memset(dmabuf->virt, 0, hw_page_size);
		eq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
					putPaddrLow(dmabuf->phys);
		eq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
					putPaddrHigh(dmabuf->phys);
	}
	mbox->vport = phba->pport;
	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
	mbox->ctx_buf = NULL;
	mbox->ctx_ndlp = NULL;
	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2500 EQ_CREATE mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
	}
	eq->type = LPFC_EQ;
	eq->subtype = LPFC_NONE;
	eq->queue_id = bf_get(lpfc_mbx_eq_create_q_id, &eq_create->u.response);
	if (eq->queue_id == 0xFFFF)
		status = -ENXIO;
	eq->host_index = 0;
	eq->notify_interval = LPFC_EQ_NOTIFY_INTRVL;
	eq->max_proc_limit = LPFC_EQ_MAX_PROC_LIMIT;

	mempool_free(mbox, phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_cq_create - Create a Completion Queue on the HBA
 * @phba: HBA structure that indicates port to create a queue on.
 * @cq: The queue structure to use to create the completion queue.
 * @eq: The event queue to bind this completion queue to.
 *
 * This function creates a completion queue, as detailed in @wq, on a port,
 * described by @phba by sending a CQ_CREATE mailbox command to the HBA.
 *
 * The @phba struct is used to send mailbox command to HBA. The @cq struct
 * is used to get the entry count and entry size that are necessary to
 * determine the number of pages to allocate and use for this queue. The @eq
 * is used to indicate which event queue to bind this completion queue to. This
 * function will send the CQ_CREATE mailbox command to the HBA to setup the
 * completion queue. This function is asynchronous and will wait for the mailbox
 * command to finish before continuing.
 *
 * On success this function will return a zero. If unable to allocate enough
 * memory this function will return -ENOMEM. If the queue create mailbox command
 * fails this function will return -ENXIO.
 **/
int
lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq,
	       struct lpfc_queue *eq, uint32_t type, uint32_t subtype)
{
	struct lpfc_mbx_cq_create *cq_create;
	struct lpfc_dmabuf *dmabuf;
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;

	/* sanity check on queue memory */
	if (!cq || !eq)
		return -ENODEV;

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_cq_create) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_CQ_CREATE,
			 length, LPFC_SLI4_MBX_EMBED);
	cq_create = &mbox->u.mqe.un.cq_create;
	shdr = (union lpfc_sli4_cfg_shdr *) &cq_create->header.cfg_shdr;
	bf_set(lpfc_mbx_cq_create_num_pages, &cq_create->u.request,
		    cq->page_count);
	bf_set(lpfc_cq_context_event, &cq_create->u.request.context, 1);
	bf_set(lpfc_cq_context_valid, &cq_create->u.request.context, 1);
	bf_set(lpfc_mbox_hdr_version, &shdr->request,
	       phba->sli4_hba.pc_sli4_params.cqv);
	if (phba->sli4_hba.pc_sli4_params.cqv == LPFC_Q_CREATE_VERSION_2) {
		bf_set(lpfc_mbx_cq_create_page_size, &cq_create->u.request,
		       (cq->page_size / SLI4_PAGE_SIZE));
		bf_set(lpfc_cq_eq_id_2, &cq_create->u.request.context,
		       eq->queue_id);
		bf_set(lpfc_cq_context_autovalid, &cq_create->u.request.context,
		       phba->sli4_hba.pc_sli4_params.cqav);
	} else {
		bf_set(lpfc_cq_eq_id, &cq_create->u.request.context,
		       eq->queue_id);
	}
	switch (cq->entry_count) {
	case 2048:
	case 4096:
		if (phba->sli4_hba.pc_sli4_params.cqv ==
		    LPFC_Q_CREATE_VERSION_2) {
			cq_create->u.request.context.lpfc_cq_context_count =
				cq->entry_count;
			bf_set(lpfc_cq_context_count,
			       &cq_create->u.request.context,
			       LPFC_CQ_CNT_WORD7);
			break;
		}
		/* fall through */
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0361 Unsupported CQ count: "
				"entry cnt %d sz %d pg cnt %d\n",
				cq->entry_count, cq->entry_size,
				cq->page_count);
		if (cq->entry_count < 256) {
			status = -EINVAL;
			goto out;
		}
		/* fall through - otherwise default to smallest count */
	case 256:
		bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
		       LPFC_CQ_CNT_256);
		break;
	case 512:
		bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
		       LPFC_CQ_CNT_512);
		break;
	case 1024:
		bf_set(lpfc_cq_context_count, &cq_create->u.request.context,
		       LPFC_CQ_CNT_1024);
		break;
	}
	list_for_each_entry(dmabuf, &cq->page_list, list) {
		memset(dmabuf->virt, 0, cq->page_size);
		cq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
					putPaddrLow(dmabuf->phys);
		cq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
					putPaddrHigh(dmabuf->phys);
	}
	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);

	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2501 CQ_CREATE mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
		goto out;
	}
	cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response);
	if (cq->queue_id == 0xFFFF) {
		status = -ENXIO;
		goto out;
	}
	/* link the cq onto the parent eq child list */
	list_add_tail(&cq->list, &eq->child_list);
	/* Set up completion queue's type and subtype */
	cq->type = type;
	cq->subtype = subtype;
	cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response);
	cq->assoc_qid = eq->queue_id;
	cq->assoc_qp = eq;
	cq->host_index = 0;
	cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL;
	cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, cq->entry_count);

	if (cq->queue_id > phba->sli4_hba.cq_max)
		phba->sli4_hba.cq_max = cq->queue_id;
out:
	mempool_free(mbox, phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_cq_create_set - Create a set of Completion Queues on the HBA for MRQ
 * @phba: HBA structure that indicates port to create a queue on.
 * @cqp: The queue structure array to use to create the completion queues.
 * @hdwq: The hardware queue array  with the EQ to bind completion queues to.
 *
 * This function creates a set of  completion queue, s to support MRQ
 * as detailed in @cqp, on a port,
 * described by @phba by sending a CREATE_CQ_SET mailbox command to the HBA.
 *
 * The @phba struct is used to send mailbox command to HBA. The @cq struct
 * is used to get the entry count and entry size that are necessary to
 * determine the number of pages to allocate and use for this queue. The @eq
 * is used to indicate which event queue to bind this completion queue to. This
 * function will send the CREATE_CQ_SET mailbox command to the HBA to setup the
 * completion queue. This function is asynchronous and will wait for the mailbox
 * command to finish before continuing.
 *
 * On success this function will return a zero. If unable to allocate enough
 * memory this function will return -ENOMEM. If the queue create mailbox command
 * fails this function will return -ENXIO.
 **/
int
lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp,
		   struct lpfc_sli4_hdw_queue *hdwq, uint32_t type,
		   uint32_t subtype)
{
	struct lpfc_queue *cq;
	struct lpfc_queue *eq;
	struct lpfc_mbx_cq_create_set *cq_set;
	struct lpfc_dmabuf *dmabuf;
	LPFC_MBOXQ_t *mbox;
	int rc, length, alloclen, status = 0;
	int cnt, idx, numcq, page_idx = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;

	/* sanity check on queue memory */
	numcq = phba->cfg_nvmet_mrq;
	if (!cqp || !hdwq || !numcq)
		return -ENODEV;

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;

	length = sizeof(struct lpfc_mbx_cq_create_set);
	length += ((numcq * cqp[0]->page_count) *
		   sizeof(struct dma_address));
	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			LPFC_MBOX_OPCODE_FCOE_CQ_CREATE_SET, length,
			LPFC_SLI4_MBX_NEMBED);
	if (alloclen < length) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"3098 Allocated DMA memory size (%d) is "
				"less than the requested DMA memory size "
				"(%d)\n", alloclen, length);
		status = -ENOMEM;
		goto out;
	}
	cq_set = mbox->sge_array->addr[0];
	shdr = (union lpfc_sli4_cfg_shdr *)&cq_set->cfg_shdr;
	bf_set(lpfc_mbox_hdr_version, &shdr->request, 0);

	for (idx = 0; idx < numcq; idx++) {
		cq = cqp[idx];
		eq = hdwq[idx].hba_eq;
		if (!cq || !eq) {
			status = -ENOMEM;
			goto out;
		}
		if (!phba->sli4_hba.pc_sli4_params.supported)
			hw_page_size = cq->page_size;

		switch (idx) {
		case 0:
			bf_set(lpfc_mbx_cq_create_set_page_size,
			       &cq_set->u.request,
			       (hw_page_size / SLI4_PAGE_SIZE));
			bf_set(lpfc_mbx_cq_create_set_num_pages,
			       &cq_set->u.request, cq->page_count);
			bf_set(lpfc_mbx_cq_create_set_evt,
			       &cq_set->u.request, 1);
			bf_set(lpfc_mbx_cq_create_set_valid,
			       &cq_set->u.request, 1);
			bf_set(lpfc_mbx_cq_create_set_cqe_size,
			       &cq_set->u.request, 0);
			bf_set(lpfc_mbx_cq_create_set_num_cq,
			       &cq_set->u.request, numcq);
			bf_set(lpfc_mbx_cq_create_set_autovalid,
			       &cq_set->u.request,
			       phba->sli4_hba.pc_sli4_params.cqav);
			switch (cq->entry_count) {
			case 2048:
			case 4096:
				if (phba->sli4_hba.pc_sli4_params.cqv ==
				    LPFC_Q_CREATE_VERSION_2) {
					bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
					       &cq_set->u.request,
						cq->entry_count);
					bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
					       &cq_set->u.request,
					       LPFC_CQ_CNT_WORD7);
					break;
				}
				/* fall through */
			default:
				lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
						"3118 Bad CQ count. (%d)\n",
						cq->entry_count);
				if (cq->entry_count < 256) {
					status = -EINVAL;
					goto out;
				}
				/* fall through - otherwise default to smallest */
			case 256:
				bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
				       &cq_set->u.request, LPFC_CQ_CNT_256);
				break;
			case 512:
				bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
				       &cq_set->u.request, LPFC_CQ_CNT_512);
				break;
			case 1024:
				bf_set(lpfc_mbx_cq_create_set_cqe_cnt,
				       &cq_set->u.request, LPFC_CQ_CNT_1024);
				break;
			}
			bf_set(lpfc_mbx_cq_create_set_eq_id0,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 1:
			bf_set(lpfc_mbx_cq_create_set_eq_id1,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 2:
			bf_set(lpfc_mbx_cq_create_set_eq_id2,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 3:
			bf_set(lpfc_mbx_cq_create_set_eq_id3,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 4:
			bf_set(lpfc_mbx_cq_create_set_eq_id4,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 5:
			bf_set(lpfc_mbx_cq_create_set_eq_id5,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 6:
			bf_set(lpfc_mbx_cq_create_set_eq_id6,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 7:
			bf_set(lpfc_mbx_cq_create_set_eq_id7,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 8:
			bf_set(lpfc_mbx_cq_create_set_eq_id8,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 9:
			bf_set(lpfc_mbx_cq_create_set_eq_id9,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 10:
			bf_set(lpfc_mbx_cq_create_set_eq_id10,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 11:
			bf_set(lpfc_mbx_cq_create_set_eq_id11,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 12:
			bf_set(lpfc_mbx_cq_create_set_eq_id12,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 13:
			bf_set(lpfc_mbx_cq_create_set_eq_id13,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 14:
			bf_set(lpfc_mbx_cq_create_set_eq_id14,
			       &cq_set->u.request, eq->queue_id);
			break;
		case 15:
			bf_set(lpfc_mbx_cq_create_set_eq_id15,
			       &cq_set->u.request, eq->queue_id);
			break;
		}

		/* link the cq onto the parent eq child list */
		list_add_tail(&cq->list, &eq->child_list);
		/* Set up completion queue's type and subtype */
		cq->type = type;
		cq->subtype = subtype;
		cq->assoc_qid = eq->queue_id;
		cq->assoc_qp = eq;
		cq->host_index = 0;
		cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL;
		cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit,
					 cq->entry_count);
		cq->chann = idx;

		rc = 0;
		list_for_each_entry(dmabuf, &cq->page_list, list) {
			memset(dmabuf->virt, 0, hw_page_size);
			cnt = page_idx + dmabuf->buffer_tag;
			cq_set->u.request.page[cnt].addr_lo =
					putPaddrLow(dmabuf->phys);
			cq_set->u.request.page[cnt].addr_hi =
					putPaddrHigh(dmabuf->phys);
			rc++;
		}
		page_idx += rc;
	}

	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);

	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"3119 CQ_CREATE_SET mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
		goto out;
	}
	rc = bf_get(lpfc_mbx_cq_create_set_base_id, &cq_set->u.response);
	if (rc == 0xFFFF) {
		status = -ENXIO;
		goto out;
	}

	for (idx = 0; idx < numcq; idx++) {
		cq = cqp[idx];
		cq->queue_id = rc + idx;
		if (cq->queue_id > phba->sli4_hba.cq_max)
			phba->sli4_hba.cq_max = cq->queue_id;
	}

out:
	lpfc_sli4_mbox_cmd_free(phba, mbox);
	return status;
}

/**
 * lpfc_mq_create_fb_init - Send MCC_CREATE without async events registration
 * @phba: HBA structure that indicates port to create a queue on.
 * @mq: The queue structure to use to create the mailbox queue.
 * @mbox: An allocated pointer to type LPFC_MBOXQ_t
 * @cq: The completion queue to associate with this cq.
 *
 * This function provides failback (fb) functionality when the
 * mq_create_ext fails on older FW generations.  It's purpose is identical
 * to mq_create_ext otherwise.
 *
 * This routine cannot fail as all attributes were previously accessed and
 * initialized in mq_create_ext.
 **/
static void
lpfc_mq_create_fb_init(struct lpfc_hba *phba, struct lpfc_queue *mq,
		       LPFC_MBOXQ_t *mbox, struct lpfc_queue *cq)
{
	struct lpfc_mbx_mq_create *mq_create;
	struct lpfc_dmabuf *dmabuf;
	int length;

	length = (sizeof(struct lpfc_mbx_mq_create) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_MQ_CREATE,
			 length, LPFC_SLI4_MBX_EMBED);
	mq_create = &mbox->u.mqe.un.mq_create;
	bf_set(lpfc_mbx_mq_create_num_pages, &mq_create->u.request,
	       mq->page_count);
	bf_set(lpfc_mq_context_cq_id, &mq_create->u.request.context,
	       cq->queue_id);
	bf_set(lpfc_mq_context_valid, &mq_create->u.request.context, 1);
	switch (mq->entry_count) {
	case 16:
		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
		       LPFC_MQ_RING_SIZE_16);
		break;
	case 32:
		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
		       LPFC_MQ_RING_SIZE_32);
		break;
	case 64:
		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
		       LPFC_MQ_RING_SIZE_64);
		break;
	case 128:
		bf_set(lpfc_mq_context_ring_size, &mq_create->u.request.context,
		       LPFC_MQ_RING_SIZE_128);
		break;
	}
	list_for_each_entry(dmabuf, &mq->page_list, list) {
		mq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
			putPaddrLow(dmabuf->phys);
		mq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
			putPaddrHigh(dmabuf->phys);
	}
}

/**
 * lpfc_mq_create - Create a mailbox Queue on the HBA
 * @phba: HBA structure that indicates port to create a queue on.
 * @mq: The queue structure to use to create the mailbox queue.
 * @cq: The completion queue to associate with this cq.
 * @subtype: The queue's subtype.
 *
 * This function creates a mailbox queue, as detailed in @mq, on a port,
 * described by @phba by sending a MQ_CREATE mailbox command to the HBA.
 *
 * The @phba struct is used to send mailbox command to HBA. The @cq struct
 * is used to get the entry count and entry size that are necessary to
 * determine the number of pages to allocate and use for this queue. This
 * function will send the MQ_CREATE mailbox command to the HBA to setup the
 * mailbox queue. This function is asynchronous and will wait for the mailbox
 * command to finish before continuing.
 *
 * On success this function will return a zero. If unable to allocate enough
 * memory this function will return -ENOMEM. If the queue create mailbox command
 * fails this function will return -ENXIO.
 **/
int32_t
lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq,
	       struct lpfc_queue *cq, uint32_t subtype)
{
	struct lpfc_mbx_mq_create *mq_create;
	struct lpfc_mbx_mq_create_ext *mq_create_ext;
	struct lpfc_dmabuf *dmabuf;
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;

	/* sanity check on queue memory */
	if (!mq || !cq)
		return -ENODEV;
	if (!phba->sli4_hba.pc_sli4_params.supported)
		hw_page_size = SLI4_PAGE_SIZE;

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_mq_create_ext) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_MQ_CREATE_EXT,
			 length, LPFC_SLI4_MBX_EMBED);

	mq_create_ext = &mbox->u.mqe.un.mq_create_ext;
	shdr = (union lpfc_sli4_cfg_shdr *) &mq_create_ext->header.cfg_shdr;
	bf_set(lpfc_mbx_mq_create_ext_num_pages,
	       &mq_create_ext->u.request, mq->page_count);
	bf_set(lpfc_mbx_mq_create_ext_async_evt_link,
	       &mq_create_ext->u.request, 1);
	bf_set(lpfc_mbx_mq_create_ext_async_evt_fip,
	       &mq_create_ext->u.request, 1);
	bf_set(lpfc_mbx_mq_create_ext_async_evt_group5,
	       &mq_create_ext->u.request, 1);
	bf_set(lpfc_mbx_mq_create_ext_async_evt_fc,
	       &mq_create_ext->u.request, 1);
	bf_set(lpfc_mbx_mq_create_ext_async_evt_sli,
	       &mq_create_ext->u.request, 1);
	bf_set(lpfc_mq_context_valid, &mq_create_ext->u.request.context, 1);
	bf_set(lpfc_mbox_hdr_version, &shdr->request,
	       phba->sli4_hba.pc_sli4_params.mqv);
	if (phba->sli4_hba.pc_sli4_params.mqv == LPFC_Q_CREATE_VERSION_1)
		bf_set(lpfc_mbx_mq_create_ext_cq_id, &mq_create_ext->u.request,
		       cq->queue_id);
	else
		bf_set(lpfc_mq_context_cq_id, &mq_create_ext->u.request.context,
		       cq->queue_id);
	switch (mq->entry_count) {
	default:
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0362 Unsupported MQ count. (%d)\n",
				mq->entry_count);
		if (mq->entry_count < 16) {
			status = -EINVAL;
			goto out;
		}
		/* fall through - otherwise default to smallest count */
	case 16:
		bf_set(lpfc_mq_context_ring_size,
		       &mq_create_ext->u.request.context,
		       LPFC_MQ_RING_SIZE_16);
		break;
	case 32:
		bf_set(lpfc_mq_context_ring_size,
		       &mq_create_ext->u.request.context,
		       LPFC_MQ_RING_SIZE_32);
		break;
	case 64:
		bf_set(lpfc_mq_context_ring_size,
		       &mq_create_ext->u.request.context,
		       LPFC_MQ_RING_SIZE_64);
		break;
	case 128:
		bf_set(lpfc_mq_context_ring_size,
		       &mq_create_ext->u.request.context,
		       LPFC_MQ_RING_SIZE_128);
		break;
	}
	list_for_each_entry(dmabuf, &mq->page_list, list) {
		memset(dmabuf->virt, 0, hw_page_size);
		mq_create_ext->u.request.page[dmabuf->buffer_tag].addr_lo =
					putPaddrLow(dmabuf->phys);
		mq_create_ext->u.request.page[dmabuf->buffer_tag].addr_hi =
					putPaddrHigh(dmabuf->phys);
	}
	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	mq->queue_id = bf_get(lpfc_mbx_mq_create_q_id,
			      &mq_create_ext->u.response);
	if (rc != MBX_SUCCESS) {
		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
				"2795 MQ_CREATE_EXT failed with "
				"status x%x. Failback to MQ_CREATE.\n",
				rc);
		lpfc_mq_create_fb_init(phba, mq, mbox, cq);
		mq_create = &mbox->u.mqe.un.mq_create;
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
		shdr = (union lpfc_sli4_cfg_shdr *) &mq_create->header.cfg_shdr;
		mq->queue_id = bf_get(lpfc_mbx_mq_create_q_id,
				      &mq_create->u.response);
	}

	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2502 MQ_CREATE mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
		goto out;
	}
	if (mq->queue_id == 0xFFFF) {
		status = -ENXIO;
		goto out;
	}
	mq->type = LPFC_MQ;
	mq->assoc_qid = cq->queue_id;
	mq->subtype = subtype;
	mq->host_index = 0;
	mq->hba_index = 0;

	/* link the mq onto the parent cq child list */
	list_add_tail(&mq->list, &cq->child_list);
out:
	mempool_free(mbox, phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_wq_create - Create a Work Queue on the HBA
 * @phba: HBA structure that indicates port to create a queue on.
 * @wq: The queue structure to use to create the work queue.
 * @cq: The completion queue to bind this work queue to.
 * @subtype: The subtype of the work queue indicating its functionality.
 *
 * This function creates a work queue, as detailed in @wq, on a port, described
 * by @phba by sending a WQ_CREATE mailbox command to the HBA.
 *
 * The @phba struct is used to send mailbox command to HBA. The @wq struct
 * is used to get the entry count and entry size that are necessary to
 * determine the number of pages to allocate and use for this queue. The @cq
 * is used to indicate which completion queue to bind this work queue to. This
 * function will send the WQ_CREATE mailbox command to the HBA to setup the
 * work queue. This function is asynchronous and will wait for the mailbox
 * command to finish before continuing.
 *
 * On success this function will return a zero. If unable to allocate enough
 * memory this function will return -ENOMEM. If the queue create mailbox command
 * fails this function will return -ENXIO.
 **/
int
lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,
	       struct lpfc_queue *cq, uint32_t subtype)
{
	struct lpfc_mbx_wq_create *wq_create;
	struct lpfc_dmabuf *dmabuf;
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
	struct dma_address *page;
	void __iomem *bar_memmap_p;
	uint32_t db_offset;
	uint16_t pci_barset;
	uint8_t dpp_barset;
	uint32_t dpp_offset;
	unsigned long pg_addr;
	uint8_t wq_create_version;

	/* sanity check on queue memory */
	if (!wq || !cq)
		return -ENODEV;
	if (!phba->sli4_hba.pc_sli4_params.supported)
		hw_page_size = wq->page_size;

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_wq_create) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			 LPFC_MBOX_OPCODE_FCOE_WQ_CREATE,
			 length, LPFC_SLI4_MBX_EMBED);
	wq_create = &mbox->u.mqe.un.wq_create;
	shdr = (union lpfc_sli4_cfg_shdr *) &wq_create->header.cfg_shdr;
	bf_set(lpfc_mbx_wq_create_num_pages, &wq_create->u.request,
		    wq->page_count);
	bf_set(lpfc_mbx_wq_create_cq_id, &wq_create->u.request,
		    cq->queue_id);

	/* wqv is the earliest version supported, NOT the latest */
	bf_set(lpfc_mbox_hdr_version, &shdr->request,
	       phba->sli4_hba.pc_sli4_params.wqv);

	if ((phba->sli4_hba.pc_sli4_params.wqsize & LPFC_WQ_SZ128_SUPPORT) ||
	    (wq->page_size > SLI4_PAGE_SIZE))
		wq_create_version = LPFC_Q_CREATE_VERSION_1;
	else
		wq_create_version = LPFC_Q_CREATE_VERSION_0;


	if (phba->sli4_hba.pc_sli4_params.wqsize & LPFC_WQ_SZ128_SUPPORT)
		wq_create_version = LPFC_Q_CREATE_VERSION_1;
	else
		wq_create_version = LPFC_Q_CREATE_VERSION_0;

	switch (wq_create_version) {
	case LPFC_Q_CREATE_VERSION_1:
		bf_set(lpfc_mbx_wq_create_wqe_count, &wq_create->u.request_1,
		       wq->entry_count);
		bf_set(lpfc_mbox_hdr_version, &shdr->request,
		       LPFC_Q_CREATE_VERSION_1);

		switch (wq->entry_size) {
		default:
		case 64:
			bf_set(lpfc_mbx_wq_create_wqe_size,
			       &wq_create->u.request_1,
			       LPFC_WQ_WQE_SIZE_64);
			break;
		case 128:
			bf_set(lpfc_mbx_wq_create_wqe_size,
			       &wq_create->u.request_1,
			       LPFC_WQ_WQE_SIZE_128);
			break;
		}
		/* Request DPP by default */
		bf_set(lpfc_mbx_wq_create_dpp_req, &wq_create->u.request_1, 1);
		bf_set(lpfc_mbx_wq_create_page_size,
		       &wq_create->u.request_1,
		       (wq->page_size / SLI4_PAGE_SIZE));
		page = wq_create->u.request_1.page;
		break;
	default:
		page = wq_create->u.request.page;
		break;
	}

	list_for_each_entry(dmabuf, &wq->page_list, list) {
		memset(dmabuf->virt, 0, hw_page_size);
		page[dmabuf->buffer_tag].addr_lo = putPaddrLow(dmabuf->phys);
		page[dmabuf->buffer_tag].addr_hi = putPaddrHigh(dmabuf->phys);
	}

	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
		bf_set(lpfc_mbx_wq_create_dua, &wq_create->u.request, 1);

	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2503 WQ_CREATE mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
		goto out;
	}

	if (wq_create_version == LPFC_Q_CREATE_VERSION_0)
		wq->queue_id = bf_get(lpfc_mbx_wq_create_q_id,
					&wq_create->u.response);
	else
		wq->queue_id = bf_get(lpfc_mbx_wq_create_v1_q_id,
					&wq_create->u.response_1);

	if (wq->queue_id == 0xFFFF) {
		status = -ENXIO;
		goto out;
	}

	wq->db_format = LPFC_DB_LIST_FORMAT;
	if (wq_create_version == LPFC_Q_CREATE_VERSION_0) {
		if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
			wq->db_format = bf_get(lpfc_mbx_wq_create_db_format,
					       &wq_create->u.response);
			if ((wq->db_format != LPFC_DB_LIST_FORMAT) &&
			    (wq->db_format != LPFC_DB_RING_FORMAT)) {
				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
						"3265 WQ[%d] doorbell format "
						"not supported: x%x\n",
						wq->queue_id, wq->db_format);
				status = -EINVAL;
				goto out;
			}
			pci_barset = bf_get(lpfc_mbx_wq_create_bar_set,
					    &wq_create->u.response);
			bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
								   pci_barset);
			if (!bar_memmap_p) {
				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
						"3263 WQ[%d] failed to memmap "
						"pci barset:x%x\n",
						wq->queue_id, pci_barset);
				status = -ENOMEM;
				goto out;
			}
			db_offset = wq_create->u.response.doorbell_offset;
			if ((db_offset != LPFC_ULP0_WQ_DOORBELL) &&
			    (db_offset != LPFC_ULP1_WQ_DOORBELL)) {
				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
						"3252 WQ[%d] doorbell offset "
						"not supported: x%x\n",
						wq->queue_id, db_offset);
				status = -EINVAL;
				goto out;
			}
			wq->db_regaddr = bar_memmap_p + db_offset;
			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
					"3264 WQ[%d]: barset:x%x, offset:x%x, "
					"format:x%x\n", wq->queue_id,
					pci_barset, db_offset, wq->db_format);
		} else
			wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
	} else {
		/* Check if DPP was honored by the firmware */
		wq->dpp_enable = bf_get(lpfc_mbx_wq_create_dpp_rsp,
				    &wq_create->u.response_1);
		if (wq->dpp_enable) {
			pci_barset = bf_get(lpfc_mbx_wq_create_v1_bar_set,
					    &wq_create->u.response_1);
			bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
								   pci_barset);
			if (!bar_memmap_p) {
				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
						"3267 WQ[%d] failed to memmap "
						"pci barset:x%x\n",
						wq->queue_id, pci_barset);
				status = -ENOMEM;
				goto out;
			}
			db_offset = wq_create->u.response_1.doorbell_offset;
			wq->db_regaddr = bar_memmap_p + db_offset;
			wq->dpp_id = bf_get(lpfc_mbx_wq_create_dpp_id,
					    &wq_create->u.response_1);
			dpp_barset = bf_get(lpfc_mbx_wq_create_dpp_bar,
					    &wq_create->u.response_1);
			bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
								   dpp_barset);
			if (!bar_memmap_p) {
				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
						"3268 WQ[%d] failed to memmap "
						"pci barset:x%x\n",
						wq->queue_id, dpp_barset);
				status = -ENOMEM;
				goto out;
			}
			dpp_offset = wq_create->u.response_1.dpp_offset;
			wq->dpp_regaddr = bar_memmap_p + dpp_offset;
			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
					"3271 WQ[%d]: barset:x%x, offset:x%x, "
					"dpp_id:x%x dpp_barset:x%x "
					"dpp_offset:x%x\n",
					wq->queue_id, pci_barset, db_offset,
					wq->dpp_id, dpp_barset, dpp_offset);

			/* Enable combined writes for DPP aperture */
			pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK;
#ifdef CONFIG_X86
			rc = set_memory_wc(pg_addr, 1);
			if (rc) {
				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"3272 Cannot setup Combined "
					"Write on WQ[%d] - disable DPP\n",
					wq->queue_id);
				phba->cfg_enable_dpp = 0;
			}
#else
			phba->cfg_enable_dpp = 0;
#endif
		} else
			wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
	}
	wq->pring = kzalloc(sizeof(struct lpfc_sli_ring), GFP_KERNEL);
	if (wq->pring == NULL) {
		status = -ENOMEM;
		goto out;
	}
	wq->type = LPFC_WQ;
	wq->assoc_qid = cq->queue_id;
	wq->subtype = subtype;
	wq->host_index = 0;
	wq->hba_index = 0;
	wq->notify_interval = LPFC_WQ_NOTIFY_INTRVL;

	/* link the wq onto the parent cq child list */
	list_add_tail(&wq->list, &cq->child_list);
out:
	mempool_free(mbox, phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_rq_create - Create a Receive Queue on the HBA
 * @phba: HBA structure that indicates port to create a queue on.
 * @hrq: The queue structure to use to create the header receive queue.
 * @drq: The queue structure to use to create the data receive queue.
 * @cq: The completion queue to bind this work queue to.
 *
 * This function creates a receive buffer queue pair , as detailed in @hrq and
 * @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
 * to the HBA.
 *
 * The @phba struct is used to send mailbox command to HBA. The @drq and @hrq
 * struct is used to get the entry count that is necessary to determine the
 * number of pages to use for this queue. The @cq is used to indicate which
 * completion queue to bind received buffers that are posted to these queues to.
 * This function will send the RQ_CREATE mailbox command to the HBA to setup the
 * receive queue pair. This function is asynchronous and will wait for the
 * mailbox command to finish before continuing.
 *
 * On success this function will return a zero. If unable to allocate enough
 * memory this function will return -ENOMEM. If the queue create mailbox command
 * fails this function will return -ENXIO.
 **/
int
lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq,
	       struct lpfc_queue *drq, struct lpfc_queue *cq, uint32_t subtype)
{
	struct lpfc_mbx_rq_create *rq_create;
	struct lpfc_dmabuf *dmabuf;
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
	void __iomem *bar_memmap_p;
	uint32_t db_offset;
	uint16_t pci_barset;

	/* sanity check on queue memory */
	if (!hrq || !drq || !cq)
		return -ENODEV;
	if (!phba->sli4_hba.pc_sli4_params.supported)
		hw_page_size = SLI4_PAGE_SIZE;

	if (hrq->entry_count != drq->entry_count)
		return -EINVAL;
	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_rq_create) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			 LPFC_MBOX_OPCODE_FCOE_RQ_CREATE,
			 length, LPFC_SLI4_MBX_EMBED);
	rq_create = &mbox->u.mqe.un.rq_create;
	shdr = (union lpfc_sli4_cfg_shdr *) &rq_create->header.cfg_shdr;
	bf_set(lpfc_mbox_hdr_version, &shdr->request,
	       phba->sli4_hba.pc_sli4_params.rqv);
	if (phba->sli4_hba.pc_sli4_params.rqv == LPFC_Q_CREATE_VERSION_1) {
		bf_set(lpfc_rq_context_rqe_count_1,
		       &rq_create->u.request.context,
		       hrq->entry_count);
		rq_create->u.request.context.buffer_size = LPFC_HDR_BUF_SIZE;
		bf_set(lpfc_rq_context_rqe_size,
		       &rq_create->u.request.context,
		       LPFC_RQE_SIZE_8);
		bf_set(lpfc_rq_context_page_size,
		       &rq_create->u.request.context,
		       LPFC_RQ_PAGE_SIZE_4096);
	} else {
		switch (hrq->entry_count) {
		default:
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"2535 Unsupported RQ count. (%d)\n",
					hrq->entry_count);
			if (hrq->entry_count < 512) {
				status = -EINVAL;
				goto out;
			}
			/* fall through - otherwise default to smallest count */
		case 512:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_512);
			break;
		case 1024:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_1024);
			break;
		case 2048:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_2048);
			break;
		case 4096:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_4096);
			break;
		}
		bf_set(lpfc_rq_context_buf_size, &rq_create->u.request.context,
		       LPFC_HDR_BUF_SIZE);
	}
	bf_set(lpfc_rq_context_cq_id, &rq_create->u.request.context,
	       cq->queue_id);
	bf_set(lpfc_mbx_rq_create_num_pages, &rq_create->u.request,
	       hrq->page_count);
	list_for_each_entry(dmabuf, &hrq->page_list, list) {
		memset(dmabuf->virt, 0, hw_page_size);
		rq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
					putPaddrLow(dmabuf->phys);
		rq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
					putPaddrHigh(dmabuf->phys);
	}
	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
		bf_set(lpfc_mbx_rq_create_dua, &rq_create->u.request, 1);

	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2504 RQ_CREATE mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
		goto out;
	}
	hrq->queue_id = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
	if (hrq->queue_id == 0xFFFF) {
		status = -ENXIO;
		goto out;
	}

	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
		hrq->db_format = bf_get(lpfc_mbx_rq_create_db_format,
					&rq_create->u.response);
		if ((hrq->db_format != LPFC_DB_LIST_FORMAT) &&
		    (hrq->db_format != LPFC_DB_RING_FORMAT)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"3262 RQ [%d] doorbell format not "
					"supported: x%x\n", hrq->queue_id,
					hrq->db_format);
			status = -EINVAL;
			goto out;
		}

		pci_barset = bf_get(lpfc_mbx_rq_create_bar_set,
				    &rq_create->u.response);
		bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset);
		if (!bar_memmap_p) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"3269 RQ[%d] failed to memmap pci "
					"barset:x%x\n", hrq->queue_id,
					pci_barset);
			status = -ENOMEM;
			goto out;
		}

		db_offset = rq_create->u.response.doorbell_offset;
		if ((db_offset != LPFC_ULP0_RQ_DOORBELL) &&
		    (db_offset != LPFC_ULP1_RQ_DOORBELL)) {
			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
					"3270 RQ[%d] doorbell offset not "
					"supported: x%x\n", hrq->queue_id,
					db_offset);
			status = -EINVAL;
			goto out;
		}
		hrq->db_regaddr = bar_memmap_p + db_offset;
		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
				"3266 RQ[qid:%d]: barset:x%x, offset:x%x, "
				"format:x%x\n", hrq->queue_id, pci_barset,
				db_offset, hrq->db_format);
	} else {
		hrq->db_format = LPFC_DB_RING_FORMAT;
		hrq->db_regaddr = phba->sli4_hba.RQDBregaddr;
	}
	hrq->type = LPFC_HRQ;
	hrq->assoc_qid = cq->queue_id;
	hrq->subtype = subtype;
	hrq->host_index = 0;
	hrq->hba_index = 0;
	hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;

	/* now create the data queue */
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			 LPFC_MBOX_OPCODE_FCOE_RQ_CREATE,
			 length, LPFC_SLI4_MBX_EMBED);
	bf_set(lpfc_mbox_hdr_version, &shdr->request,
	       phba->sli4_hba.pc_sli4_params.rqv);
	if (phba->sli4_hba.pc_sli4_params.rqv == LPFC_Q_CREATE_VERSION_1) {
		bf_set(lpfc_rq_context_rqe_count_1,
		       &rq_create->u.request.context, hrq->entry_count);
		if (subtype == LPFC_NVMET)
			rq_create->u.request.context.buffer_size =
				LPFC_NVMET_DATA_BUF_SIZE;
		else
			rq_create->u.request.context.buffer_size =
				LPFC_DATA_BUF_SIZE;
		bf_set(lpfc_rq_context_rqe_size, &rq_create->u.request.context,
		       LPFC_RQE_SIZE_8);
		bf_set(lpfc_rq_context_page_size, &rq_create->u.request.context,
		       (PAGE_SIZE/SLI4_PAGE_SIZE));
	} else {
		switch (drq->entry_count) {
		default:
			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
					"2536 Unsupported RQ count. (%d)\n",
					drq->entry_count);
			if (drq->entry_count < 512) {
				status = -EINVAL;
				goto out;
			}
			/* fall through - otherwise default to smallest count */
		case 512:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_512);
			break;
		case 1024:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_1024);
			break;
		case 2048:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_2048);
			break;
		case 4096:
			bf_set(lpfc_rq_context_rqe_count,
			       &rq_create->u.request.context,
			       LPFC_RQ_RING_SIZE_4096);
			break;
		}
		if (subtype == LPFC_NVMET)
			bf_set(lpfc_rq_context_buf_size,
			       &rq_create->u.request.context,
			       LPFC_NVMET_DATA_BUF_SIZE);
		else
			bf_set(lpfc_rq_context_buf_size,
			       &rq_create->u.request.context,
			       LPFC_DATA_BUF_SIZE);
	}
	bf_set(lpfc_rq_context_cq_id, &rq_create->u.request.context,
	       cq->queue_id);
	bf_set(lpfc_mbx_rq_create_num_pages, &rq_create->u.request,
	       drq->page_count);
	list_for_each_entry(dmabuf, &drq->page_list, list) {
		rq_create->u.request.page[dmabuf->buffer_tag].addr_lo =
					putPaddrLow(dmabuf->phys);
		rq_create->u.request.page[dmabuf->buffer_tag].addr_hi =
					putPaddrHigh(dmabuf->phys);
	}
	if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE)
		bf_set(lpfc_mbx_rq_create_dua, &rq_create->u.request, 1);
	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr = (union lpfc_sli4_cfg_shdr *) &rq_create->header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		status = -ENXIO;
		goto out;
	}
	drq->queue_id = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
	if (drq->queue_id == 0xFFFF) {
		status = -ENXIO;
		goto out;
	}
	drq->type = LPFC_DRQ;
	drq->assoc_qid = cq->queue_id;
	drq->subtype = subtype;
	drq->host_index = 0;
	drq->hba_index = 0;
	drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;

	/* link the header and data RQs onto the parent cq child list */
	list_add_tail(&hrq->list, &cq->child_list);
	list_add_tail(&drq->list, &cq->child_list);

out:
	mempool_free(mbox, phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_mrq_create - Create MRQ Receive Queues on the HBA
 * @phba: HBA structure that indicates port to create a queue on.
 * @hrqp: The queue structure array to use to create the header receive queues.
 * @drqp: The queue structure array to use to create the data receive queues.
 * @cqp: The completion queue array to bind these receive queues to.
 *
 * This function creates a receive buffer queue pair , as detailed in @hrq and
 * @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command
 * to the HBA.
 *
 * The @phba struct is used to send mailbox command to HBA. The @drq and @hrq
 * struct is used to get the entry count that is necessary to determine the
 * number of pages to use for this queue. The @cq is used to indicate which
 * completion queue to bind received buffers that are posted to these queues to.
 * This function will send the RQ_CREATE mailbox command to the HBA to setup the
 * receive queue pair. This function is asynchronous and will wait for the
 * mailbox command to finish before continuing.
 *
 * On success this function will return a zero. If unable to allocate enough
 * memory this function will return -ENOMEM. If the queue create mailbox command
 * fails this function will return -ENXIO.
 **/
int
lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp,
		struct lpfc_queue **drqp, struct lpfc_queue **cqp,
		uint32_t subtype)
{
	struct lpfc_queue *hrq, *drq, *cq;
	struct lpfc_mbx_rq_create_v2 *rq_create;
	struct lpfc_dmabuf *dmabuf;
	LPFC_MBOXQ_t *mbox;
	int rc, length, alloclen, status = 0;
	int cnt, idx, numrq, page_idx = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;
	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;

	numrq = phba->cfg_nvmet_mrq;
	/* sanity check on array memory */
	if (!hrqp || !drqp || !cqp || !numrq)
		return -ENODEV;
	if (!phba->sli4_hba.pc_sli4_params.supported)
		hw_page_size = SLI4_PAGE_SIZE;

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;

	length = sizeof(struct lpfc_mbx_rq_create_v2);
	length += ((2 * numrq * hrqp[0]->page_count) *
		   sizeof(struct dma_address));

	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
				    LPFC_MBOX_OPCODE_FCOE_RQ_CREATE, length,
				    LPFC_SLI4_MBX_NEMBED);
	if (alloclen < length) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"3099 Allocated DMA memory size (%d) is "
				"less than the requested DMA memory size "
				"(%d)\n", alloclen, length);
		status = -ENOMEM;
		goto out;
	}



	rq_create = mbox->sge_array->addr[0];
	shdr = (union lpfc_sli4_cfg_shdr *)&rq_create->cfg_shdr;

	bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_Q_CREATE_VERSION_2);
	cnt = 0;

	for (idx = 0; idx < numrq; idx++) {
		hrq = hrqp[idx];
		drq = drqp[idx];
		cq  = cqp[idx];

		/* sanity check on queue memory */
		if (!hrq || !drq || !cq) {
			status = -ENODEV;
			goto out;
		}

		if (hrq->entry_count != drq->entry_count) {
			status = -EINVAL;
			goto out;
		}

		if (idx == 0) {
			bf_set(lpfc_mbx_rq_create_num_pages,
			       &rq_create->u.request,
			       hrq->page_count);
			bf_set(lpfc_mbx_rq_create_rq_cnt,
			       &rq_create->u.request, (numrq * 2));
			bf_set(lpfc_mbx_rq_create_dnb, &rq_create->u.request,
			       1);
			bf_set(lpfc_rq_context_base_cq,
			       &rq_create->u.request.context,
			       cq->queue_id);
			bf_set(lpfc_rq_context_data_size,
			       &rq_create->u.request.context,
			       LPFC_NVMET_DATA_BUF_SIZE);
			bf_set(lpfc_rq_context_hdr_size,
			       &rq_create->u.request.context,
			       LPFC_HDR_BUF_SIZE);
			bf_set(lpfc_rq_context_rqe_count_1,
			       &rq_create->u.request.context,
			       hrq->entry_count);
			bf_set(lpfc_rq_context_rqe_size,
			       &rq_create->u.request.context,
			       LPFC_RQE_SIZE_8);
			bf_set(lpfc_rq_context_page_size,
			       &rq_create->u.request.context,
			       (PAGE_SIZE/SLI4_PAGE_SIZE));
		}
		rc = 0;
		list_for_each_entry(dmabuf, &hrq->page_list, list) {
			memset(dmabuf->virt, 0, hw_page_size);
			cnt = page_idx + dmabuf->buffer_tag;
			rq_create->u.request.page[cnt].addr_lo =
					putPaddrLow(dmabuf->phys);
			rq_create->u.request.page[cnt].addr_hi =
					putPaddrHigh(dmabuf->phys);
			rc++;
		}
		page_idx += rc;

		rc = 0;
		list_for_each_entry(dmabuf, &drq->page_list, list) {
			memset(dmabuf->virt, 0, hw_page_size);
			cnt = page_idx + dmabuf->buffer_tag;
			rq_create->u.request.page[cnt].addr_lo =
					putPaddrLow(dmabuf->phys);
			rq_create->u.request.page[cnt].addr_hi =
					putPaddrHigh(dmabuf->phys);
			rc++;
		}
		page_idx += rc;

		hrq->db_format = LPFC_DB_RING_FORMAT;
		hrq->db_regaddr = phba->sli4_hba.RQDBregaddr;
		hrq->type = LPFC_HRQ;
		hrq->assoc_qid = cq->queue_id;
		hrq->subtype = subtype;
		hrq->host_index = 0;
		hrq->hba_index = 0;
		hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;

		drq->db_format = LPFC_DB_RING_FORMAT;
		drq->db_regaddr = phba->sli4_hba.RQDBregaddr;
		drq->type = LPFC_DRQ;
		drq->assoc_qid = cq->queue_id;
		drq->subtype = subtype;
		drq->host_index = 0;
		drq->hba_index = 0;
		drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL;

		list_add_tail(&hrq->list, &cq->child_list);
		list_add_tail(&drq->list, &cq->child_list);
	}

	rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"3120 RQ_CREATE mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
		goto out;
	}
	rc = bf_get(lpfc_mbx_rq_create_q_id, &rq_create->u.response);
	if (rc == 0xFFFF) {
		status = -ENXIO;
		goto out;
	}

	/* Initialize all RQs with associated queue id */
	for (idx = 0; idx < numrq; idx++) {
		hrq = hrqp[idx];
		hrq->queue_id = rc + (2 * idx);
		drq = drqp[idx];
		drq->queue_id = rc + (2 * idx) + 1;
	}

out:
	lpfc_sli4_mbox_cmd_free(phba, mbox);
	return status;
}

/**
 * lpfc_eq_destroy - Destroy an event Queue on the HBA
 * @eq: The queue structure associated with the queue to destroy.
 *
 * This function destroys a queue, as detailed in @eq by sending an mailbox
 * command, specific to the type of queue, to the HBA.
 *
 * The @eq struct is used to get the queue ID of the queue to destroy.
 *
 * On success this function will return a zero. If the queue destroy mailbox
 * command fails this function will return -ENXIO.
 **/
int
lpfc_eq_destroy(struct lpfc_hba *phba, struct lpfc_queue *eq)
{
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;

	/* sanity check on queue memory */
	if (!eq)
		return -ENODEV;

	mbox = mempool_alloc(eq->phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_eq_destroy) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_EQ_DESTROY,
			 length, LPFC_SLI4_MBX_EMBED);
	bf_set(lpfc_mbx_eq_destroy_q_id, &mbox->u.mqe.un.eq_destroy.u.request,
	       eq->queue_id);
	mbox->vport = eq->phba->pport;
	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;

	rc = lpfc_sli_issue_mbox(eq->phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr = (union lpfc_sli4_cfg_shdr *)
		&mbox->u.mqe.un.eq_destroy.header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2505 EQ_DESTROY mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
	}

	/* Remove eq from any list */
	list_del_init(&eq->list);
	mempool_free(mbox, eq->phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_cq_destroy - Destroy a Completion Queue on the HBA
 * @cq: The queue structure associated with the queue to destroy.
 *
 * This function destroys a queue, as detailed in @cq by sending an mailbox
 * command, specific to the type of queue, to the HBA.
 *
 * The @cq struct is used to get the queue ID of the queue to destroy.
 *
 * On success this function will return a zero. If the queue destroy mailbox
 * command fails this function will return -ENXIO.
 **/
int
lpfc_cq_destroy(struct lpfc_hba *phba, struct lpfc_queue *cq)
{
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;

	/* sanity check on queue memory */
	if (!cq)
		return -ENODEV;
	mbox = mempool_alloc(cq->phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_cq_destroy) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_CQ_DESTROY,
			 length, LPFC_SLI4_MBX_EMBED);
	bf_set(lpfc_mbx_cq_destroy_q_id, &mbox->u.mqe.un.cq_destroy.u.request,
	       cq->queue_id);
	mbox->vport = cq->phba->pport;
	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
	rc = lpfc_sli_issue_mbox(cq->phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr = (union lpfc_sli4_cfg_shdr *)
		&mbox->u.mqe.un.wq_create.header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2506 CQ_DESTROY mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
	}
	/* Remove cq from any list */
	list_del_init(&cq->list);
	mempool_free(mbox, cq->phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_mq_destroy - Destroy a Mailbox Queue on the HBA
 * @qm: The queue structure associated with the queue to destroy.
 *
 * This function destroys a queue, as detailed in @mq by sending an mailbox
 * command, specific to the type of queue, to the HBA.
 *
 * The @mq struct is used to get the queue ID of the queue to destroy.
 *
 * On success this function will return a zero. If the queue destroy mailbox
 * command fails this function will return -ENXIO.
 **/
int
lpfc_mq_destroy(struct lpfc_hba *phba, struct lpfc_queue *mq)
{
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;

	/* sanity check on queue memory */
	if (!mq)
		return -ENODEV;
	mbox = mempool_alloc(mq->phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_mq_destroy) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
			 LPFC_MBOX_OPCODE_MQ_DESTROY,
			 length, LPFC_SLI4_MBX_EMBED);
	bf_set(lpfc_mbx_mq_destroy_q_id, &mbox->u.mqe.un.mq_destroy.u.request,
	       mq->queue_id);
	mbox->vport = mq->phba->pport;
	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
	rc = lpfc_sli_issue_mbox(mq->phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr = (union lpfc_sli4_cfg_shdr *)
		&mbox->u.mqe.un.mq_destroy.header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2507 MQ_DESTROY mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
	}
	/* Remove mq from any list */
	list_del_init(&mq->list);
	mempool_free(mbox, mq->phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_wq_destroy - Destroy a Work Queue on the HBA
 * @wq: The queue structure associated with the queue to destroy.
 *
 * This function destroys a queue, as detailed in @wq by sending an mailbox
 * command, specific to the type of queue, to the HBA.
 *
 * The @wq struct is used to get the queue ID of the queue to destroy.
 *
 * On success this function will return a zero. If the queue destroy mailbox
 * command fails this function will return -ENXIO.
 **/
int
lpfc_wq_destroy(struct lpfc_hba *phba, struct lpfc_queue *wq)
{
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;

	/* sanity check on queue memory */
	if (!wq)
		return -ENODEV;
	mbox = mempool_alloc(wq->phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_wq_destroy) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			 LPFC_MBOX_OPCODE_FCOE_WQ_DESTROY,
			 length, LPFC_SLI4_MBX_EMBED);
	bf_set(lpfc_mbx_wq_destroy_q_id, &mbox->u.mqe.un.wq_destroy.u.request,
	       wq->queue_id);
	mbox->vport = wq->phba->pport;
	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
	rc = lpfc_sli_issue_mbox(wq->phba, mbox, MBX_POLL);
	shdr = (union lpfc_sli4_cfg_shdr *)
		&mbox->u.mqe.un.wq_destroy.header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2508 WQ_DESTROY mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
	}
	/* Remove wq from any list */
	list_del_init(&wq->list);
	kfree(wq->pring);
	wq->pring = NULL;
	mempool_free(mbox, wq->phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_rq_destroy - Destroy a Receive Queue on the HBA
 * @rq: The queue structure associated with the queue to destroy.
 *
 * This function destroys a queue, as detailed in @rq by sending an mailbox
 * command, specific to the type of queue, to the HBA.
 *
 * The @rq struct is used to get the queue ID of the queue to destroy.
 *
 * On success this function will return a zero. If the queue destroy mailbox
 * command fails this function will return -ENXIO.
 **/
int
lpfc_rq_destroy(struct lpfc_hba *phba, struct lpfc_queue *hrq,
		struct lpfc_queue *drq)
{
	LPFC_MBOXQ_t *mbox;
	int rc, length, status = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;

	/* sanity check on queue memory */
	if (!hrq || !drq)
		return -ENODEV;
	mbox = mempool_alloc(hrq->phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;
	length = (sizeof(struct lpfc_mbx_rq_destroy) -
		  sizeof(struct lpfc_sli4_cfg_mhdr));
	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			 LPFC_MBOX_OPCODE_FCOE_RQ_DESTROY,
			 length, LPFC_SLI4_MBX_EMBED);
	bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request,
	       hrq->queue_id);
	mbox->vport = hrq->phba->pport;
	mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
	rc = lpfc_sli_issue_mbox(hrq->phba, mbox, MBX_POLL);
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr = (union lpfc_sli4_cfg_shdr *)
		&mbox->u.mqe.un.rq_destroy.header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2509 RQ_DESTROY mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		if (rc != MBX_TIMEOUT)
			mempool_free(mbox, hrq->phba->mbox_mem_pool);
		return -ENXIO;
	}
	bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request,
	       drq->queue_id);
	rc = lpfc_sli_issue_mbox(drq->phba, mbox, MBX_POLL);
	shdr = (union lpfc_sli4_cfg_shdr *)
		&mbox->u.mqe.un.rq_destroy.header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2510 RQ_DESTROY mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		status = -ENXIO;
	}
	list_del_init(&hrq->list);
	list_del_init(&drq->list);
	mempool_free(mbox, hrq->phba->mbox_mem_pool);
	return status;
}

/**
 * lpfc_sli4_post_sgl - Post scatter gather list for an XRI to HBA
 * @phba: The virtual port for which this call being executed.
 * @pdma_phys_addr0: Physical address of the 1st SGL page.
 * @pdma_phys_addr1: Physical address of the 2nd SGL page.
 * @xritag: the xritag that ties this io to the SGL pages.
 *
 * This routine will post the sgl pages for the IO that has the xritag
 * that is in the iocbq structure. The xritag is assigned during iocbq
 * creation and persists for as long as the driver is loaded.
 * if the caller has fewer than 256 scatter gather segments to map then
 * pdma_phys_addr1 should be 0.
 * If the caller needs to map more than 256 scatter gather segment then
 * pdma_phys_addr1 should be a valid physical address.
 * physical address for SGLs must be 64 byte aligned.
 * If you are going to map 2 SGL's then the first one must have 256 entries
 * the second sgl can have between 1 and 256 entries.
 *
 * Return codes:
 * 	0 - Success
 * 	-ENXIO, -ENOMEM - Failure
 **/
int
lpfc_sli4_post_sgl(struct lpfc_hba *phba,
		dma_addr_t pdma_phys_addr0,
		dma_addr_t pdma_phys_addr1,
		uint16_t xritag)
{
	struct lpfc_mbx_post_sgl_pages *post_sgl_pages;
	LPFC_MBOXQ_t *mbox;
	int rc;
	uint32_t shdr_status, shdr_add_status;
	uint32_t mbox_tmo;
	union lpfc_sli4_cfg_shdr *shdr;

	if (xritag == NO_XRI) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"0364 Invalid param:\n");
		return -EINVAL;
	}

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;

	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES,
			sizeof(struct lpfc_mbx_post_sgl_pages) -
			sizeof(struct lpfc_sli4_cfg_mhdr), LPFC_SLI4_MBX_EMBED);

	post_sgl_pages = (struct lpfc_mbx_post_sgl_pages *)
				&mbox->u.mqe.un.post_sgl_pages;
	bf_set(lpfc_post_sgl_pages_xri, post_sgl_pages, xritag);
	bf_set(lpfc_post_sgl_pages_xricnt, post_sgl_pages, 1);

	post_sgl_pages->sgl_pg_pairs[0].sgl_pg0_addr_lo	=
				cpu_to_le32(putPaddrLow(pdma_phys_addr0));
	post_sgl_pages->sgl_pg_pairs[0].sgl_pg0_addr_hi =
				cpu_to_le32(putPaddrHigh(pdma_phys_addr0));

	post_sgl_pages->sgl_pg_pairs[0].sgl_pg1_addr_lo	=
				cpu_to_le32(putPaddrLow(pdma_phys_addr1));
	post_sgl_pages->sgl_pg_pairs[0].sgl_pg1_addr_hi =
				cpu_to_le32(putPaddrHigh(pdma_phys_addr1));
	if (!phba->sli4_hba.intr_enable)
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	else {
		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
	}
	/* The IOCTL status is embedded in the mailbox subheader. */
	shdr = (union lpfc_sli4_cfg_shdr *) &post_sgl_pages->header.cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (rc != MBX_TIMEOUT)
		mempool_free(mbox, phba->mbox_mem_pool);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2511 POST_SGL mailbox failed with "
				"status x%x add_status x%x, mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
	}
	return 0;
}

/**
 * lpfc_sli4_alloc_xri - Get an available rpi in the device's range
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine is invoked to post rpi header templates to the
 * HBA consistent with the SLI-4 interface spec.  This routine
 * posts a SLI4_PAGE_SIZE memory region to the port to hold up to
 * SLI4_PAGE_SIZE modulo 64 rpi context headers.
 *
 * Returns
 *	A nonzero rpi defined as rpi_base <= rpi < max_rpi if successful
 *	LPFC_RPI_ALLOC_ERROR if no rpis are available.
 **/
static uint16_t
lpfc_sli4_alloc_xri(struct lpfc_hba *phba)
{
	unsigned long xri;

	/*
	 * Fetch the next logical xri.  Because this index is logical,
	 * the driver starts at 0 each time.
	 */
	spin_lock_irq(&phba->hbalock);
	xri = find_next_zero_bit(phba->sli4_hba.xri_bmask,
				 phba->sli4_hba.max_cfg_param.max_xri, 0);
	if (xri >= phba->sli4_hba.max_cfg_param.max_xri) {
		spin_unlock_irq(&phba->hbalock);
		return NO_XRI;
	} else {
		set_bit(xri, phba->sli4_hba.xri_bmask);
		phba->sli4_hba.max_cfg_param.xri_used++;
	}
	spin_unlock_irq(&phba->hbalock);
	return xri;
}

/**
 * lpfc_sli4_free_xri - Release an xri for reuse.
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine is invoked to release an xri to the pool of
 * available rpis maintained by the driver.
 **/
static void
__lpfc_sli4_free_xri(struct lpfc_hba *phba, int xri)
{
	if (test_and_clear_bit(xri, phba->sli4_hba.xri_bmask)) {
		phba->sli4_hba.max_cfg_param.xri_used--;
	}
}

/**
 * lpfc_sli4_free_xri - Release an xri for reuse.
 * @phba: pointer to lpfc hba data structure.
 *
 * This routine is invoked to release an xri to the pool of
 * available rpis maintained by the driver.
 **/
void
lpfc_sli4_free_xri(struct lpfc_hba *phba, int xri)
{
	spin_lock_irq(&phba->hbalock);
	__lpfc_sli4_free_xri(phba, xri);
	spin_unlock_irq(&phba->hbalock);
}

/**
 * lpfc_sli4_next_xritag - Get an xritag for the io
 * @phba: Pointer to HBA context object.
 *
 * This function gets an xritag for the iocb. If there is no unused xritag
 * it will return 0xffff.
 * The function returns the allocated xritag if successful, else returns zero.
 * Zero is not a valid xritag.
 * The caller is not required to hold any lock.
 **/
uint16_t
lpfc_sli4_next_xritag(struct lpfc_hba *phba)
{
	uint16_t xri_index;

	xri_index = lpfc_sli4_alloc_xri(phba);
	if (xri_index == NO_XRI)
		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
				"2004 Failed to allocate XRI.last XRITAG is %d"
				" Max XRI is %d, Used XRI is %d\n",
				xri_index,
				phba->sli4_hba.max_cfg_param.max_xri,
				phba->sli4_hba.max_cfg_param.xri_used);
	return xri_index;
}

/**
 * lpfc_sli4_post_sgl_list - post a block of ELS sgls to the port.
 * @phba: pointer to lpfc hba data structure.
 * @post_sgl_list: pointer to els sgl entry list.
 * @count: number of els sgl entries on the list.
 *
 * This routine is invoked to post a block of driver's sgl pages to the
 * HBA using non-embedded mailbox command. No Lock is held. This routine
 * is only called when the driver is loading and after all IO has been
 * stopped.
 **/
static int
lpfc_sli4_post_sgl_list(struct lpfc_hba *phba,
			    struct list_head *post_sgl_list,
			    int post_cnt)
{
	struct lpfc_sglq *sglq_entry = NULL, *sglq_next = NULL;
	struct lpfc_mbx_post_uembed_sgl_page1 *sgl;
	struct sgl_page_pairs *sgl_pg_pairs;
	void *viraddr;
	LPFC_MBOXQ_t *mbox;
	uint32_t reqlen, alloclen, pg_pairs;
	uint32_t mbox_tmo;
	uint16_t xritag_start = 0;
	int rc = 0;
	uint32_t shdr_status, shdr_add_status;
	union lpfc_sli4_cfg_shdr *shdr;

	reqlen = post_cnt * sizeof(struct sgl_page_pairs) +
		 sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
	if (reqlen > SLI4_PAGE_SIZE) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"2559 Block sgl registration required DMA "
				"size (%d) great than a page\n", reqlen);
		return -ENOMEM;
	}

	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox)
		return -ENOMEM;

	/* Allocate DMA memory and set up the non-embedded mailbox command */
	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
			 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen,
			 LPFC_SLI4_MBX_NEMBED);

	if (alloclen < reqlen) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"0285 Allocated DMA memory size (%d) is "
				"less than the requested DMA memory "
				"size (%d)\n", alloclen, reqlen);
		lpfc_sli4_mbox_cmd_free(phba, mbox);
		return -ENOMEM;
	}
	/* Set up the SGL pages in the non-embedded DMA pages */
	viraddr = mbox->sge_array->addr[0];
	sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr;
	sgl_pg_pairs = &sgl->sgl_pg_pairs;

	pg_pairs = 0;
	list_for_each_entry_safe(sglq_entry, sglq_next, post_sgl_list, list) {
		/* Set up the sge entry */
		sgl_pg_pairs->sgl_pg0_addr_lo =
				cpu_to_le32(putPaddrLow(sglq_entry->phys));
		sgl_pg_pairs->sgl_pg0_addr_hi =
				cpu_to_le32(putPaddrHigh(sglq_entry->phys));
		sgl_pg_pairs->sgl_pg1_addr_lo =
				cpu_to_le32(putPaddrLow(0));
		sgl_pg_pairs->sgl_pg1_addr_hi =
				cpu_to_le32(putPaddrHigh(0));

		/* Keep the first xritag on the list */
		if (pg_pairs == 0)
			xritag_start = sglq_entry->sli4_xritag;
		sgl_pg_pairs++;
		pg_pairs++;
	}

	/* Complete initialization and perform endian conversion. */
	bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
	bf_set(lpfc_post_sgl_pages_xricnt, sgl, post_cnt);
	sgl->word0 = cpu_to_le32(sgl->word0);

	if (!phba->sli4_hba.intr_enable)
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	else {
		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
	}
	shdr = (union lpfc_sli4_cfg_shdr *) &sgl->cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (rc != MBX_TIMEOUT)
		lpfc_sli4_mbox_cmd_free(phba, mbox);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"2513 POST_SGL_BLOCK mailbox command failed "
				"status x%x add_status x%x mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		rc = -ENXIO;
	}
	return rc;
}

/**
 * lpfc_sli4_post_io_sgl_block - post a block of nvme sgl list to firmware
 * @phba: pointer to lpfc hba data structure.
 * @nblist: pointer to nvme buffer list.
 * @count: number of scsi buffers on the list.
 *
 * This routine is invoked to post a block of @count scsi sgl pages from a
 * SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
 * No Lock is held.
 *
 **/
static int
lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist,
			    int count)
{
	struct lpfc_io_buf *lpfc_ncmd;
	struct lpfc_mbx_post_uembed_sgl_page1 *sgl;
	struct sgl_page_pairs *sgl_pg_pairs;
	void *viraddr;
	LPFC_MBOXQ_t *mbox;
	uint32_t reqlen, alloclen, pg_pairs;
	uint32_t mbox_tmo;
	uint16_t xritag_start = 0;
	int rc = 0;
	uint32_t shdr_status, shdr_add_status;
	dma_addr_t pdma_phys_bpl1;
	union lpfc_sli4_cfg_shdr *shdr;

	/* Calculate the requested length of the dma memory */
	reqlen = count * sizeof(struct sgl_page_pairs) +
		 sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t);
	if (reqlen > SLI4_PAGE_SIZE) {
		lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
				"6118 Block sgl registration required DMA "
				"size (%d) great than a page\n", reqlen);
		return -ENOMEM;
	}
	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
	if (!mbox) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"6119 Failed to allocate mbox cmd memory\n");
		return -ENOMEM;
	}

	/* Allocate DMA memory and set up the non-embedded mailbox command */
	alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,
				    LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES,
				    reqlen, LPFC_SLI4_MBX_NEMBED);

	if (alloclen < reqlen) {
		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
				"6120 Allocated DMA memory size (%d) is "
				"less than the requested DMA memory "
				"size (%d)\n", alloclen, reqlen);
		lpfc_sli4_mbox_cmd_free(phba, mbox);
		return -ENOMEM;
	}

	/* Get the first SGE entry from the non-embedded DMA memory */
	viraddr = mbox->sge_array->addr[0];

	/* Set up the SGL pages in the non-embedded DMA pages */
	sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr;
	sgl_pg_pairs = &sgl->sgl_pg_pairs;

	pg_pairs = 0;
	list_for_each_entry(lpfc_ncmd, nblist, list) {
		/* Set up the sge entry */
		sgl_pg_pairs->sgl_pg0_addr_lo =
			cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl));
		sgl_pg_pairs->sgl_pg0_addr_hi =
			cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl));
		if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE)
			pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl +
						SGL_PAGE_SIZE;
		else
			pdma_phys_bpl1 = 0;
		sgl_pg_pairs->sgl_pg1_addr_lo =
			cpu_to_le32(putPaddrLow(pdma_phys_bpl1));
		sgl_pg_pairs->sgl_pg1_addr_hi =
			cpu_to_le32(putPaddrHigh(pdma_phys_bpl1));
		/* Keep the first xritag on the list */
		if (pg_pairs == 0)
			xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag;
		sgl_pg_pairs++;
		pg_pairs++;
	}
	bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start);
	bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs);
	/* Perform endian conversion if necessary */
	sgl->word0 = cpu_to_le32(sgl->word0);

	if (!phba->sli4_hba.intr_enable) {
		rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
	} else {
		mbox_tmo = lpfc_mbox_tmo_val(phba, mbox);
		rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo);
	}
	shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr;
	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
	if (rc != MBX_TIMEOUT)
		lpfc_sli4_mbox_cmd_free(phba, mbox);
	if (shdr_status || shdr_add_status || rc) {
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
				"6125 POST_SGL_BLOCK mailbox command failed "
				"status x%x add_status x%x mbx status x%x\n",
				shdr_status, shdr_add_status, rc);
		rc = -ENXIO;
	}
	return rc;
}

/**
 * lpfc_sli4_post_io_sgl_list - Post blocks of nvme buffer sgls from a list
 * @phba: pointer to lpfc hba data structure.
 * @post_nblist: pointer to the nvme buffer list.
 *
 * This routine walks a list of nvme buffers that was passed in. It attempts
 * to construct blocks of nvme buffer sgls which contains contiguous xris and
 * uses the non-embedded SGL block post mailbox commands to post to the port.
 * For single NVME buffer sgl with non-contiguous xri, if any, it shall use
 * embedded SGL post mailbox command for posting. The @post_nblist passed in
 * must be local list, thus no lock is needed when manipulate the list.
 *
 * Returns: 0 = failure, non-zero number of successfully posted buffers.
 **/
int
lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba,
			   struct list_head *post_nblist, int sb_count)
{
	struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next;
	int status, sgl_size;
	int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0;
	dma_addr_t pdma_phys_sgl1;
	int last_xritag = NO_XRI;
	int cur_xritag;
	LIST_HEAD(prep_nblist);
	LIST_HEAD(blck_nblist);
	LIST_HEAD(nvme_nblist);

	/* sanity check */
	if (sb_count <= 0)
		return -EINVAL;

	sgl_size = phba->cfg_sg_dma_buf_size;
	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) {
		list_del_init(&lpfc_ncmd->list);
		block_cnt++;
		if ((last_xritag != NO_XRI) &&
		    (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) {
			/* a hole in xri block, form a sgl posting block */
			list_splice_init(&prep_nblist, &blck_nblist);
			post_cnt = block_cnt - 1;
			/* prepare list for next posting block */
			list_add_tail(&lpfc_ncmd->list, &prep_nblist);
			block_cnt = 1;
		} else {
			/* prepare list for next posting block */
			list_add_tail(&lpfc_ncmd->list, &prep_nblist);
			/* enough sgls for non-embed sgl mbox command */
			if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) {
				list_splice_init(&prep_nblist, &blck_nblist);
				post_cnt = block_cnt;
				block_cnt = 0;
			}
		}
		num_posting++;
		last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;

		/* end of repost sgl list condition for NVME buffers */
		if (num_posting == sb_count) {
			if (post_cnt == 0) {
				/* last sgl posting block */
				list_splice_init(&prep_nblist, &blck_nblist);
				post_cnt = block_cnt;
			} else if (block_cnt == 1) {
				/* last single sgl with non-contiguous xri */
				if (sgl_size > SGL_PAGE_SIZE)
					pdma_phys_sgl1 =
						lpfc_ncmd->dma_phys_sgl +
						SGL_PAGE_SIZE;
				else
					pdma_phys_sgl1 = 0;
				cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;
				status = lpfc_sli4_post_sgl(
						phba, lpfc_ncmd->dma_phys_sgl,
						pdma_phys_sgl1, cur_xritag);
				if (status) {
					/* Post error.  Buffer unavailable. */
					lpfc_ncmd->flags |=
						LPFC_SBUF_NOT_POSTED;
				} else {
					/* Post success. Bffer available. */
					lpfc_ncmd->flags &=
						~LPFC_SBUF_NOT_POSTED;
					lpfc_ncmd->status = IOSTAT_SUCCESS;
					num_posted++;
				}
				/* success, put on NVME buffer sgl list */
				list_add_tail(&lpfc_ncmd->list, &nvme_nblist);
			}
		}

		/* continue until a nembed page worth of sgls */
		if (post_cnt == 0)
			continue;

		/* post block of NVME buffer list sgls */
		status = lpfc_sli4_post_io_sgl_block(phba, &blck_nblist,
						     post_cnt);

		/* don't reset xirtag due to hole in xri block */
		if (block_cnt == 0)
			last_xritag = NO_XRI;

		/* reset NVME buffer post count for next round of posting */
		post_cnt = 0;

		/* put posted NVME buffer-sgl posted on NVME buffer sgl list */
		while (!list_empty(&blck_nblist)) {
			list_remove_head(&blck_nblist, lpfc_ncmd,
					 struct lpfc_io_buf, list);
			if (status) {
				/* Post error.  Mark buffer unavailable. */
				lpfc_ncmd->flags |= LPFC_SBUF_NOT_POSTED;
			} else {
				/* Post success, Mark buffer available. */
				lpfc_ncmd->flags &= ~LPFC_SBUF_NOT_POSTED;
				lpfc_ncmd->status = IOSTAT_SUCCESS;
				num_posted++;
			}
			list_add_tail(&lpfc_ncmd->list, &nvme_nblist);
		}
	}
	/* Push NVME buffers with sgl posted to the available list */
	lpfc_io_buf_replenish(phba, &nvme_nblist);

	return num_posted;
}

/**
 * lpfc_fc_frame_check - Check that this frame is a valid frame to handle
 * @phba: pointer to lpfc_hba struct that the frame was received on
 * @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
 *
 * This function checks the fields in the @fc_hdr to see if the FC frame is a
 * valid type of frame that the LPFC driver will handle. This function will
 * return a zero if the frame is a valid frame or a non zero value when the
 * frame does not pass the check.
 **/
static int
lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr)
{
	/*  make rctl_names static to save stack space */
	struct fc_vft_header *fc_vft_hdr;
	uint32_t *header = (uint32_t *) fc_hdr;

	switch (fc_hdr->fh_r_ctl) {
	case FC_RCTL_DD_UNCAT:		/* uncategorized information */
	case FC_RCTL_DD_SOL_DATA:	/* solicited data */
	case FC_RCTL_DD_UNSOL_CTL:	/* unsolicited control */
	case FC_RCTL_DD_SOL_CTL:	/* solicited control or reply */
	case FC_RCTL_DD_UNSOL_DATA:	/* unsolicited data */
	case FC_RCTL_DD_DATA_DESC:	/* data descriptor */
	case FC_RCTL_DD_UNSOL_CMD:	/* unsolicited command */
	case FC_RCTL_DD_CMD_STATUS:	/* command status */
	case FC_RCTL_ELS_REQ:	/* extended link services request */
	case FC_RCTL_ELS_REP:	/* extended link services reply */
	case FC_RCTL_ELS4_REQ:	/* FC-4 ELS request */
	case FC_RCTL_ELS4_REP:	/* FC-4 ELS reply */
	case FC_RCTL_BA_NOP:  	/* basic link service NOP */
	case FC_RCTL_BA_ABTS: 	/* basic link service abort */
	case FC_RCTL_BA_RMC: 	/* remove connection */
	case FC_RCTL_BA_ACC:	/* basic accept */
	case FC_RCTL_BA_RJT:	/* basic reject */
	case FC_RCTL_BA_PRMT:
	case FC_RCTL_ACK_1:	/* acknowledge_1 */
	case FC_RCTL_ACK_0:	/* acknowledge_0 */
	case FC_RCTL_P_RJT:	/* port reject */
	case FC_RCTL_F_RJT:	/* fabric reject */
	case FC_RCTL_P_BSY:	/* port busy */
	case FC_RCTL_F_BSY:	/* fabric busy to data frame */
	case FC_RCTL_F_BSYL:	/* fabric busy to link control frame */
	case FC_RCTL_LCR:	/* link credit reset */
	case FC_RCTL_MDS_DIAGS: /* MDS Diagnostics */
	case FC_RCTL_END:	/* end */
		break;
	case FC_RCTL_VFTH:	/* Virtual Fabric tagging Header */
		fc_vft_hdr = (struct fc_vft_header *)fc_hdr;
		fc_hdr = &((struct fc_frame_header *)fc_vft_hdr)[1];
		return lpfc_fc_frame_check(phba, fc_hdr);
	default:
		goto drop;
	}

	switch (fc_hdr->fh_type) {
	case FC_TYPE_BLS:
	case FC_TYPE_ELS:
	case FC_TYPE_FCP:
	case FC_TYPE_CT:
	case FC_TYPE_NVME:
		break;
	case FC_TYPE_IP:
	case FC_TYPE_ILS:
	default:
		goto drop;
	}

	lpfc_printf_log(phba, KERN_INFO, LOG_ELS,
			"2538 Received frame rctl:x%x, type:x%x, "
			"frame Data:%08x %08x %08x %08x %08x %08x %08x\n",
			fc_hdr->fh_r_ctl, fc_hdr->fh_type,
			be32_to_cpu(header[0]), be32_to_cpu(header[1]),
			be32_to_cpu(header[2]), be32_to_cpu(header[3]),
			be32_to_cpu(header[4]), be32_to_cpu(header[5]),
			be32_to_cpu(header[6]));
	return 0;
drop:
	lpfc_printf_log(phba, KERN_WARNING, LOG_ELS,
			"2539 Dropped frame rctl:x%x type:x%x\n",
			fc_hdr->fh_r_ctl, fc_hdr->fh_type);
	return 1;
}

/**
 * lpfc_fc_hdr_get_vfi - Get the VFI from an FC frame
 * @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
 *
 * This function processes the FC header to retrieve the VFI from the VF
 * header, if one exists. This function will return the VFI if one exists
 * or 0 if no VSAN Header exists.
 **/
static uint32_t
lpfc_fc_hdr_get_vfi(struct fc_frame_header *fc_hdr)
{
	struct fc_vft_header *fc_vft_hdr = (struct fc_vft_header *)fc_hdr;

	if (fc_hdr->fh_r_ctl != FC_RCTL_VFTH)
		return 0;
	return bf_get(fc_vft_hdr_vf_id, fc_vft_hdr);
}

/**
 * lpfc_fc_frame_to_vport - Finds the vport that a frame is destined to
 * @phba: Pointer to the HBA structure to search for the vport on
 * @fc_hdr: A pointer to the FC Header data (In Big Endian Format)
 * @fcfi: The FC Fabric ID that the frame came from
 *
 * This function searches the @phba for a vport that matches the content of the
 * @fc_hdr passed in and the @fcfi. This function uses the @fc_hdr to fetch the
 * VFI, if the Virtual Fabric Tagging Header exists, and the DID. This function
 * returns the matching vport pointer or NULL if unable to match frame to a
 * vport.
 **/
static struct lpfc_vport *
lpfc_fc_frame_to_vport(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr,
		       uint16_t fcfi, uint32_t did)
{
	struct lpfc_vport **vports;
	struct lpfc_vport *vport = NULL;
	int i;

	if (did == Fabric_DID)
		return phba->pport;
	if ((phba->pport->fc_flag & FC_PT2PT) &&
		!(phba->link_state == LPFC_HBA_READY))
		return phba->pport;

	vports = lpfc_create_vport_work_array(phba);
	if (vports != NULL) {
		for (i = 0; i <= phba->max_vpi && vports[i] != NULL; i++) {
			if (phba->fcf.fcfi == fcfi &&
			    vports[i]->vfi == lpfc_fc_hdr_get_vfi(fc_hdr) &&
			    vports[i]->fc_myDID == did) {
				vport = vports[i];
				break;
			}
		}
	}
	lpfc_destroy_vport_work_array(phba, vports);
	return vport;
}

/**
 * lpfc_update_rcv_time_stamp - Update vport's rcv seq time stamp
 * @vport: The vport to work on.
 *
 * This function updates the receive sequence time stamp for this vport. The
 * receive sequence time stamp indicates the time that the last frame of the
 * the sequence that has been idle for the longest amount of time was received.
 * the driver uses this time stamp to indicate if any received sequences have
 * timed out.
 **/
static void
lpfc_update_rcv_time_stamp(struct lpfc_vport *vport)
{
	struct lpfc_dmabuf *h_buf;
	struct hbq_dmabuf *dmabuf = NULL;

	/* get the oldest sequence on the rcv list */
	h_buf = list_get_first(&vport->rcv_buffer_list,
			       struct lpfc_dmabuf, list);
	if (!h_buf)
		return;
	dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
	vport->rcv_buffer_time_stamp = dmabuf->time_stamp;
}

/**
 * lpfc_cleanup_rcv_buffers - Cleans up all outstanding receive sequences.
 * @vport: The vport that the received sequences were sent to.
 *
 * This function cleans up all outstanding received sequences. This is called
 * by the driver when a link event or user action invalidates all the received
 * sequences.
 **/
void
lpfc_cleanup_rcv_buffers(struct lpfc_vport *vport)
{
	struct lpfc_dmabuf *h_buf, *hnext;
	struct lpfc_dmabuf *d_buf, *dnext;
	struct hbq_dmabuf *dmabuf = NULL;

	/* start with the oldest sequence on the rcv list */
	list_for_each_entry_safe(h_buf, hnext, &vport->rcv_buffer_list, list) {
		dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
		list_del_init(&dmabuf->hbuf.list);
		list_for_each_entry_safe(d_buf, dnext,
					 &dmabuf->dbuf.list, list) {
			list_del_init(&d_buf->list);
			lpfc_in_buf_free(vport->phba, d_buf);
		}
		lpfc_in_buf_free(vport->phba, &dmabuf->dbuf);
	}
}

/**
 * lpfc_rcv_seq_check_edtov - Cleans up timed out receive sequences.
 * @vport: The vport that the received sequences were sent to.
 *
 * This function determines whether any received sequences have timed out by
 * first checking the vport's rcv_buffer_time_stamp. If this time_stamp
 * indicates that there is at least one timed out sequence this routine will
 * go through the received sequences one at a time from most inactive to most
 * active to determine which ones need to be cleaned up. Once it has determined
 * that a sequence needs to be cleaned up it will simply free up the resources
 * without sending an abort.
 **/
void
lpfc_rcv_seq_check_edtov(struct lpfc_vport *vport)
{
	struct lpfc_dmabuf *h_buf, *hnext;
	struct lpfc_dmabuf *d_buf, *dnext;
	struct hbq_dmabuf *dmabuf = NULL;
	unsigned long timeout;
	int abort_count = 0;

	timeout = (msecs_to_jiffies(vport->phba->fc_edtov) +
		   vport->rcv_buffer_time_stamp);
	if (list_empty(&vport->rcv_buffer_list) ||
	    time_before(jiffies, timeout))
		return;
	/* start with the oldest sequence on the rcv list */
	list_for_each_entry_safe(h_buf, hnext, &vport->rcv_buffer_list, list) {
		dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
		timeout = (msecs_to_jiffies(vport->phba->fc_edtov) +
			   dmabuf->time_stamp);
		if (time_before(jiffies, timeout))
			break;
		abort_count++;
		list_del_init(&dmabuf->hbuf.list);
		list_for_each_entry_safe(d_buf, dnext,
					 &dmabuf->dbuf.list, list) {
			list_del_init(&d_buf->list);
			lpfc_in_buf_free(vport->phba, d_buf);
		}
		lpfc_in_buf_free(vport->phba, &dmabuf->dbuf);
	}
	if (abort_count)
		lpfc_update_rcv_time_stamp(vport);
}

/**
 * lpfc_fc_frame_add - Adds a frame to the vport's list of received sequences
 * @dmabuf: pointer to a dmabuf that describes the hdr and data of the FC frame
 *
 * This function searches through the existing incomplete sequences that have
 * been sent to this @vport. If the frame matches one of the incomplete
 * sequences then the dbuf in the @dmabuf is added to the list of frames that
 * make up that sequence. If no sequence is found that matches this frame then
 * the function will add the hbuf in the @dmabuf to the @vport's rcv_buffer_list
 * This function returns a pointer to the first dmabuf in the sequence list that
 * the frame was linked to.
 **/
static struct hbq_dmabuf *
lpfc_fc_frame_add(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf)
{
	struct fc_frame_header *new_hdr;
	struct fc_frame_header *temp_hdr;
	struct lpfc_dmabuf *d_buf;
	struct lpfc_dmabuf *h_buf;
	struct hbq_dmabuf *seq_dmabuf = NULL;
	struct hbq_dmabuf *temp_dmabuf = NULL;
	uint8_t	found = 0;

	INIT_LIST_HEAD(&dmabuf->dbuf.list);
	dmabuf->time_stamp = jiffies;
	new_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;

	/* Use the hdr_buf to find the sequence that this frame belongs to */
	list_for_each_entry(h_buf, &vport->rcv_buffer_list, list) {
		temp_hdr = (struct fc_frame_header *)h_buf->virt;
		if ((temp_hdr->fh_seq_id != new_hdr->fh_seq_id) ||
		    (temp_hdr->fh_ox_id != new_hdr->fh_ox_id) ||
		    (memcmp(&temp_hdr->fh_s_id, &new_hdr->fh_s_id, 3)))
			continue;
		/* found a pending sequence that matches this frame */
		seq_dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
		break;
	}
	if (!seq_dmabuf) {
		/*
		 * This indicates first frame received for this sequence.
		 * Queue the buffer on the vport's rcv_buffer_list.
		 */
		list_add_tail(&dmabuf->hbuf.list, &vport->rcv_buffer_list);
		lpfc_update_rcv_time_stamp(vport);
		return dmabuf;
	}
	temp_hdr = seq_dmabuf->hbuf.virt;
	if (be16_to_cpu(new_hdr->fh_seq_cnt) <
		be16_to_cpu(temp_hdr->fh_seq_cnt)) {
		list_del_init(&seq_dmabuf->hbuf.list);
		list_add_tail(&dmabuf->hbuf.list, &vport->rcv_buffer_list);
		list_add_tail(&dmabuf->dbuf.list, &seq_dmabuf->dbuf.list);
		lpfc_update_rcv_time_stamp(vport);
		return dmabuf;
	}
	/* move this sequence to the tail to indicate a young sequence */
	list_move_tail(&seq_dmabuf->hbuf.list, &vport->rcv_buffer_list);
	seq_dmabuf->time_stamp = jiffies;
	lpfc_update_rcv_time_stamp(vport);
	if (list_empty(&seq_dmabuf->dbuf.list)) {
		temp_hdr = dmabuf->hbuf.virt;
		list_add_tail(&dmabuf->dbuf.list, &seq_dmabuf->dbuf.list);
		return seq_dmabuf;
	}
	/* find the correct place in the sequence to insert this frame */
	d_buf = list_entry(seq_dmabuf->dbuf.list.prev, typeof(*d_buf), list);
	while (!found) {
		temp_dmabuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
		temp_hdr = (struct fc_frame_header *)temp_dmabuf->hbuf.virt;
		/*
		 * If the frame's sequence count is greater than the frame on
		 * the list then insert the frame right after this frame
		 */
		if (be16_to_cpu(new_hdr->fh_seq_cnt) >
			be16_to_cpu(temp_hdr->fh_seq_cnt)) {
			list_add(&dmabuf->dbuf.list, &temp_dmabuf->dbuf.list);
			found = 1;
			break;
		}

		if (&d_buf->list == &seq_dmabuf->dbuf.list)
			break;
		d_buf = list_entry(d_buf->list.prev, typeof(*d_buf), list);
	}

	if (found)
		return seq_dmabuf;
	return NULL;
}

/**
 * lpfc_sli4_abort_partial_seq - Abort partially assembled unsol sequence
 * @vport: pointer to a vitural port
 * @dmabuf: pointer to a dmabuf that describes the FC sequence
 *
 * This function tries to abort from the partially assembed sequence, described
 * by the information from basic abbort @dmabuf. It checks to see whether such
 * partially assembled sequence held by the driver. If so, it shall free up all
 * the frames from the partially assembled sequence.
 *
 * Return
 * true  -- if there is matching partially assembled sequence present and all
 *          the frames freed with the sequence;
 * false -- if there is no matching partially assembled sequence present so
 *          nothing got aborted in the lower layer driver
 **/
static bool
lpfc_sli4_abort_partial_seq(struct lpfc_vport *vport,
			    struct hbq_dmabuf *dmabuf)
{
	struct fc_frame_header *new_hdr;
	struct fc_frame_header *temp_hdr;
	struct lpfc_dmabuf *d_buf, *n_buf, *h_buf;
	struct hbq_dmabuf *seq_dmabuf = NULL;

	/* Use the hdr_buf to find the sequence that matches this frame */
	INIT_LIST_HEAD(&dmabuf->dbuf.list);
	INIT_LIST_HEAD(&dmabuf->hbuf.list);
	new_hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
	list_for_each_entry(h_buf, &vport->rcv_buffer_list, list) {
		temp_hdr = (struct fc_frame_header *)h_buf->virt;
		if ((temp_hdr->fh_seq_id != new_hdr->fh_seq_id) ||
		    (temp_hdr->fh_ox_id != new_hdr->fh_ox_id) ||
		    (memcmp(&temp_hdr->fh_s_id, &new_hdr->fh_s_id, 3)))
			continue;
		/* found a pending sequence that matches this frame */
		seq_dmabuf = container_of(h_buf, struct hbq_dmabuf, hbuf);
		break;
	}

	/* Free up all the frames from the partially assembled sequence */
	if (seq_dmabuf) {
		list_for_each_entry_safe(d_buf, n_buf,
					 &seq_dmabuf->dbuf.list, list) {
			list_del_init(&d_buf->list);
			lpfc_in_buf_free(vport->phba, d_buf);
		}
		return true;
	}
	return false;
}

/**
 * lpfc_sli4_abort_ulp_seq - Abort assembled unsol sequence from ulp
 * @vport: pointer to a vitural port
 * @dmabuf: pointer to a dmabuf that describes the FC sequence
 *
 * This function tries to abort from the assembed sequence from upper level
 * protocol, described by the information from basic abbort @dmabuf. It
 * checks to see whether such pending context exists at upper level protocol.
 * If so, it shall clean up the pending context.
 *
 * Return
 * true  -- if there is matching pending context of the sequence cleaned
 *          at ulp;
 * false -- if there is no matching pending context of the sequence present
 *          at ulp.
 **/
static bool
lpfc_sli4_abort_ulp_seq(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf)
{
	struct lpfc_hba *phba = vport->phba;
	int handled;

	/* Accepting abort at ulp with SLI4 only */
	if (phba->sli_rev < LPFC_SLI_REV4)
		return false;

	/* Register all caring upper level protocols to attend abort */
	handled = lpfc_ct_handle_unsol_abort(phba, dmabuf);
	if (handled)
		return true;

	return false;
}

/**
 * lpfc_sli4_seq_abort_rsp_cmpl - BLS ABORT RSP seq abort iocb complete handler
 * @phba: Pointer to HBA context object.
 * @cmd_iocbq: pointer to the command iocbq structure.
 * @rsp_iocbq: pointer to the response iocbq structure.
 *
 * This function handles the sequence abort response iocb command complete
 * event. It properly releases the memory allocated to the sequence abort
 * accept iocb.
 **/
static void
lpfc_sli4_seq_abort_rsp_cmpl(struct lpfc_hba *phba,
			     struct lpfc_iocbq *cmd_iocbq,
			     struct lpfc_iocbq *rsp_iocbq)
{
	struct lpfc_nodelist *ndlp;

	if (cmd_iocbq) {
		ndlp = (struct lpfc_nodelist *)cmd_iocbq->context1;
		lpfc_nlp_put(ndlp);
		lpfc_nlp_not_used(ndlp);
		lpfc_sli_release_iocbq(phba, cmd_iocbq);
	}

	/* Failure means BLS ABORT RSP did not get delivered to remote node*/
	if (rsp_iocbq && rsp_iocbq->iocb.ulpStatus)
		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
			"3154 BLS ABORT RSP failed, data:  x%x/x%x\n",
			rsp_iocbq->iocb.ulpStatus,
			rsp_iocbq->iocb.un.ulpWord[4]);
}

/**
 * lpfc_sli4_xri_inrange - check xri is in range of xris owned by driver.
 * @phba: Pointer to HBA context object.
 * @xri: xri id in transaction.
 *
 * This function validates the xri maps to the known range of XRIs allocated an
 * used by the driver.
 **/
uint16_t
lpfc_sli4_xri_inrange(struct lpfc_hba *phba,
		      uint16_t xri)
{
	uint16_t i;

	for (i = 0; i < phba->sli4_hba.max_cfg_param.max_xri; i++) {
		if (xri == phba->sli4_hba.xri_ids[i])
			return i;
	}
	return NO_XRI;
}

/**
 * lpfc_sli4_seq_abort_rsp - bls rsp to sequence abort
 * @phba: Pointer to HBA context object.
 * @fc_hdr: pointer to a FC frame header.
 *
 * This function sends a basic response to a previous unsol sequence abort
 * event after aborting the sequence handling.
 **/
void
lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport,
			struct fc_frame_header *fc_hdr, bool aborted)
{
	struct lpfc_hba *phba = vport->phba;
	struct lpfc_iocbq *ctiocb = NULL;
	struct lpfc_nodelist *ndlp;
	uint16_t oxid, rxid, xri, lxri;
	uint32_t sid, fctl;
	IOCB_t *icmd;
	int rc;

	if (!lpfc_is_link_up(phba))
		return;

	sid = sli4_sid_from_fc_hdr(fc_hdr);
	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
	rxid = be16_to_cpu(fc_hdr->fh_rx_id);

	ndlp = lpfc_findnode_did(vport, sid);
	if (!ndlp) {
		ndlp = lpfc_nlp_init(vport, sid);
		if (!ndlp) {
			lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS,
					 "1268 Failed to allocate ndlp for "
					 "oxid:x%x SID:x%x\n", oxid, sid);
			return;
		}
		/* Put ndlp onto pport node list */
		lpfc_enqueue_node(vport, ndlp);
	} else if (!NLP_CHK_NODE_ACT(ndlp)) {
		/* re-setup ndlp without removing from node list */
		ndlp = lpfc_enable_node(vport, ndlp, NLP_STE_UNUSED_NODE);
		if (!ndlp) {
			lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS,
					 "3275 Failed to active ndlp found "
					 "for oxid:x%x SID:x%x\n", oxid, sid);
			return;
		}
	}

	/* Allocate buffer for rsp iocb */
	ctiocb = lpfc_sli_get_iocbq(phba);
	if (!ctiocb)
		return;

	/* Extract the F_CTL field from FC_HDR */
	fctl = sli4_fctl_from_fc_hdr(fc_hdr);

	icmd = &ctiocb->iocb;
	icmd->un.xseq64.bdl.bdeSize = 0;
	icmd->un.xseq64.bdl.ulpIoTag32 = 0;
	icmd->un.xseq64.w5.hcsw.Dfctl = 0;
	icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_ACC;
	icmd->un.xseq64.w5.hcsw.Type = FC_TYPE_BLS;

	/* Fill in the rest of iocb fields */
	icmd->ulpCommand = CMD_XMIT_BLS_RSP64_CX;
	icmd->ulpBdeCount = 0;
	icmd->ulpLe = 1;
	icmd->ulpClass = CLASS3;
	icmd->ulpContext = phba->sli4_hba.rpi_ids[ndlp->nlp_rpi];
	ctiocb->context1 = lpfc_nlp_get(ndlp);

	ctiocb->iocb_cmpl = NULL;
	ctiocb->vport = phba->pport;
	ctiocb->iocb_cmpl = lpfc_sli4_seq_abort_rsp_cmpl;
	ctiocb->sli4_lxritag = NO_XRI;
	ctiocb->sli4_xritag = NO_XRI;

	if (fctl & FC_FC_EX_CTX)
		/* Exchange responder sent the abort so we
		 * own the oxid.
		 */
		xri = oxid;
	else
		xri = rxid;
	lxri = lpfc_sli4_xri_inrange(phba, xri);
	if (lxri != NO_XRI)
		lpfc_set_rrq_active(phba, ndlp, lxri,
			(xri == oxid) ? rxid : oxid, 0);
	/* For BA_ABTS from exchange responder, if the logical xri with
	 * the oxid maps to the FCP XRI range, the port no longer has
	 * that exchange context, send a BLS_RJT. Override the IOCB for
	 * a BA_RJT.
	 */
	if ((fctl & FC_FC_EX_CTX) &&
	    (lxri > lpfc_sli4_get_iocb_cnt(phba))) {
		icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_RJT;
		bf_set(lpfc_vndr_code, &icmd->un.bls_rsp, 0);
		bf_set(lpfc_rsn_expln, &icmd->un.bls_rsp, FC_BA_RJT_INV_XID);
		bf_set(lpfc_rsn_code, &icmd->un.bls_rsp, FC_BA_RJT_UNABLE);
	}

	/* If BA_ABTS failed to abort a partially assembled receive sequence,
	 * the driver no longer has that exchange, send a BLS_RJT. Override
	 * the IOCB for a BA_RJT.
	 */
	if (aborted == false) {
		icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_RJT;
		bf_set(lpfc_vndr_code, &icmd->un.bls_rsp, 0);
		bf_set(lpfc_rsn_expln, &icmd->un.bls_rsp, FC_BA_RJT_INV_XID);
		bf_set(lpfc_rsn_code, &icmd->un.bls_rsp, FC_BA_RJT_UNABLE);
	}

	if (fctl & FC_FC_EX_CTX) {
		/* ABTS sent by responder to CT exchange, construction
		 * of BA_ACC will use OX_ID from ABTS for the XRI_TAG
		 * field and RX_ID from ABTS for RX_ID field.
		 */
		bf_set(lpfc_abts_orig, &icmd->un.bls_rsp, LPFC_ABTS_UNSOL_RSP);
	} else {
		/* ABTS sent by initiator to CT exchange, construction
		 * of BA_ACC will need to allocate a new XRI as for the
		 * XRI_TAG field.
		 */
		bf_set(lpfc_abts_orig, &icmd->un.bls_rsp, LPFC_ABTS_UNSOL_INT);
	}
	bf_set(lpfc_abts_rxid, &icmd->un.bls_rsp, rxid);
	bf_set(lpfc_abts_oxid, &icmd->un.bls_rsp, oxid);

	/* Xmit CT abts response on exchange <xid> */
	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
			 "1200 Send BLS cmd x%x on oxid x%x Data: x%x\n",
			 icmd->un.xseq64.w5.hcsw.Rctl, oxid, phba->link_state);

	rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, ctiocb, 0);
	if (rc == IOCB_ERROR) {
		lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
				 "2925 Failed to issue CT ABTS RSP x%x on "
				 "xri x%x, Data x%x\n",
				 icmd->un.xseq64.w5.hcsw.Rctl, oxid,
				 phba->link_state);
		lpfc_nlp_put(ndlp);
		ctiocb->context1 = NULL;
		lpfc_sli_release_iocbq(phba, ctiocb);
	}
}

/**
 * lpfc_sli4_handle_unsol_abort - Handle sli-4 unsolicited abort event
 * @vport: Pointer to the vport on which this sequence was received
 * @dmabuf: pointer to a dmabuf that describes the FC sequence
 *
 * This function handles an SLI-4 unsolicited abort event. If the unsolicited
 * receive sequence is only partially assembed by the driver, it shall abort
 * the partially assembled frames for the sequence. Otherwise, if the
 * unsolicited receive sequence has been completely assembled and passed to
 * the Upper Layer Protocol (UPL), it then mark the per oxid status for the
 * unsolicited sequence has been aborted. After that, it will issue a basic
 * accept to accept the abort.
 **/
static void
lpfc_sli4_handle_unsol_abort(struct lpfc_vport *vport,
			     struct hbq_dmabuf *dmabuf)
{
	struct lpfc_hba *phba = vport->phba;
	struct fc_frame_header fc_hdr;
	uint32_t fctl;
	bool aborted;

	/* Make a copy of fc_hdr before the dmabuf being released */
	memcpy(&fc_hdr, dmabuf->hbuf.virt, sizeof(struct fc_frame_header));
	fctl = sli4_fctl_from_fc_hdr(&fc_hdr);

	if (fctl & FC_FC_EX_CTX) {
		/* ABTS by responder to exchange, no cleanup needed */
		aborted = true;
	} else {
		/* ABTS by initiator to exchange, need to do cleanup */
		aborted = lpfc_sli4_abort_partial_seq(vport, dmabuf);
		if (aborted == false)
			aborted = lpfc_sli4_abort_ulp_seq(vport, dmabuf);
	}
	lpfc_in_buf_free(phba, &dmabuf->dbuf);

	if (phba->nvmet_support) {
		lpfc_nvmet_rcv_unsol_abort(vport, &fc_hdr);
		return;
	}

	/* Respond with BA_ACC or BA_RJT accordingly */
	lpfc_sli4_seq_abort_rsp(vport, &fc_hdr, aborted);
}

/**
 * lpfc_seq_complete - Indicates if a sequence is complete
 * @dmabuf: pointer to a dmabuf that describes the FC sequence
 *
 * This function checks the sequence, starting with the frame described by
 * @dmabuf, to see if all the frames associated with this sequence are present.
 * the frames associated with this sequence are linked to the @dmabuf using the
 * dbuf list. This function looks for two major things. 1) That the first frame
 * has a sequence count of zero. 2) There is a frame with last frame of sequence
 * set. 3) That there are no holes in the sequence count. The function will
 * return 1 when the sequence is complete, otherwise it will return 0.
 **/
static int
lpfc_seq_complete(struct hbq_dmabuf *dmabuf)
{
	struct fc_frame_header *hdr;
	struct lpfc_dmabuf *d_buf;
	struct hbq_dmabuf *seq_dmabuf;
	uint32_t fctl;
	int seq_count = 0;

	hdr = (struct fc_frame_header *)dmabuf->hbuf.virt;
	/* make sure first fame of sequence has a sequence count of zero */
	if (hdr->fh_seq_cnt != seq_count)
		return 0;
	fctl = (hdr->fh_f_ctl[0] << 16 |
		hdr->fh_f_ctl[1] << 8 |
		hdr->fh_f_ctl[2]);
	/* If last frame of sequence we can return success. */
	if (fctl & FC_FC_END_SEQ)
		return 1;
	list_for_each_entry(d_buf, &dmabuf->dbuf.list, list) {
		seq_dmabuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
		hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
		/* If there is a hole in the sequence count then fail. */
		if (++seq_count != be16_to_cpu(hdr->fh_seq_cnt))
			return 0;
		fctl = (hdr->fh_f_ctl[0] << 16 |
			hdr->fh_f_ctl[1] << 8 |
			hdr->fh_f_ctl[2]);
		/* If last frame of sequence we can return success. */
		if (fctl & FC_FC_END_SEQ)
			return 1;
	}
	return 0;
}

/**
 * lpfc_prep_seq - Prep sequence for ULP processing
 * @vport: Pointer to the vport on which this sequence was received
 * @dmabuf: pointer to a dmabuf that describes the FC sequence
 *
 * This function takes a sequence, described by a list of frames, and creates
 * a list of iocbq structures to describe the sequence. This iocbq list will be
 * used to issue to the generic unsolicited sequence handler. This routine
 * returns a pointer to the first iocbq in the list. If the function is unable
 * to allocate an iocbq then it throw out the received frames that were not
 * able to be described and return a pointer to the first iocbq. If unable to
 * allocate any iocbqs (including the first) this function will return NULL.
 **/
static struct lpfc_iocbq *
lpfc_prep_seq(struct lpfc_vport *vport, struct hbq_dmabuf *seq_dmabuf)
{
	struct hbq_dmabuf *hbq_buf;
	struct lpfc_dmabuf *d_buf, *n_buf;
	struct lpfc_iocbq *first_iocbq, *iocbq;
	struct fc_frame_header *fc_hdr;
	uint32_t sid;
	uint32_t len, tot_len;
	struct ulp_bde64 *pbde;

	fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt;
	/* remove from receive buffer list */
	list_del_init(&seq_dmabuf->hbuf.list);
	lpfc_update_rcv_time_stamp(vport);
	/* get the Remote Port's SID */
	sid = sli4_sid_from_fc_hdr(fc_hdr);
	tot_len = 0;
	/* Get an iocbq struct to fill in. */
	first_iocbq = lpfc_sli_get_iocbq(vport->phba);
	if (first_iocbq) {
		/* Initialize the first IOCB. */
		first_iocbq->iocb.unsli3.rcvsli3.acc_len = 0;
		first_iocbq->iocb.ulpStatus = IOSTAT_SUCCESS;
		first_iocbq->vport = vport;

		/* Check FC Header to see what TYPE of frame we are rcv'ing */
		if (sli4_type_from_fc_hdr(fc_hdr) == FC_TYPE_ELS) {
			first_iocbq->iocb.ulpCommand = CMD_IOCB_RCV_ELS64_CX;
			first_iocbq->iocb.un.rcvels.parmRo =
				sli4_did_from_fc_hdr(fc_hdr);
			first_iocbq->iocb.ulpPU = PARM_NPIV_DID;
		} else
			first_iocbq->iocb.ulpCommand = CMD_IOCB_RCV_SEQ64_CX;
		first_iocbq->iocb.ulpContext = NO_XRI;
		first_iocbq->iocb.unsli3.rcvsli3.ox_id =
			be16_to_cpu(fc_hdr->fh_ox_id);
		/* iocbq is prepped for internal consumption.  Physical vpi. */
		first_iocbq->iocb.unsli3.rcvsli3.vpi =
			vport->phba->vpi_ids[vport->vpi];
		/* put the first buffer into the first IOCBq */
		tot_len = bf_get(lpfc_rcqe_length,
				       &seq_dmabuf->cq_event.cqe.rcqe_cmpl);

		first_iocbq->context2 = &seq_dmabuf->dbuf;
		first_iocbq->context3 = NULL;
		first_iocbq->iocb.ulpBdeCount = 1;
		if (tot_len > LPFC_DATA_BUF_SIZE)
			first_iocbq->iocb.un.cont64[0].tus.f.bdeSize =
							LPFC_DATA_BUF_SIZE;
		else
			first_iocbq->iocb.un.cont64[0].tus.f.bdeSize = tot_len;

		first_iocbq->iocb.un.rcvels.remoteID = sid;

		first_iocbq->iocb.unsli3.rcvsli3.acc_len = tot_len;
	}
	iocbq = first_iocbq;
	/*
	 * Each IOCBq can have two Buffers assigned, so go through the list
	 * of buffers for this sequence and save two buffers in each IOCBq
	 */
	list_for_each_entry_safe(d_buf, n_buf, &seq_dmabuf->dbuf.list, list) {
		if (!iocbq) {
			lpfc_in_buf_free(vport->phba, d_buf);
			continue;
		}
		if (!iocbq->context3) {
			iocbq->context3 = d_buf;
			iocbq->iocb.ulpBdeCount++;
			/* We need to get the size out of the right CQE */
			hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
			len = bf_get(lpfc_rcqe_length,
				       &hbq_buf->cq_event.cqe.rcqe_cmpl);
			pbde = (struct ulp_bde64 *)
					&iocbq->iocb.unsli3.sli3Words[4];
			if (len > LPFC_DATA_BUF_SIZE)
				pbde->tus.f.bdeSize = LPFC_DATA_BUF_SIZE;
			else
				pbde->tus.f.bdeSize = len;

			iocbq->iocb.unsli3.rcvsli3.acc_len += len;
			tot_len += len;