summaryrefslogtreecommitdiffstats
path: root/third_party/aom/aom_dsp/mips/itrans16_dspr2.c
blob: c63b1e8570642d1f68ca9e9ab0299f282db2a7f6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "aom_dsp/mips/inv_txfm_dspr2.h"
#include "aom_dsp/txfm_common.h"

#if HAVE_DSPR2
void idct16_rows_dspr2(const int16_t *input, int16_t *output,
                       uint32_t no_rows) {
  int i;
  int step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6, step1_7;
  int step1_10, step1_11, step1_12, step1_13;
  int step2_0, step2_1, step2_2, step2_3;
  int step2_8, step2_9, step2_10, step2_11;
  int step2_12, step2_13, step2_14, step2_15;
  int load1, load2, load3, load4, load5, load6, load7, load8;
  int result1, result2, result3, result4;
  const int const_2_power_13 = 8192;

  for (i = no_rows; i--;) {
    /* prefetch row */
    prefetch_load((const uint8_t *)(input + 16));

    __asm__ __volatile__(
        "lh       %[load1],              0(%[input])                    \n\t"
        "lh       %[load2],             16(%[input])                    \n\t"
        "lh       %[load3],              8(%[input])                    \n\t"
        "lh       %[load4],             24(%[input])                    \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"
        "add      %[result1],           %[load1],       %[load2]        \n\t"
        "sub      %[result2],           %[load1],       %[load2]        \n\t"
        "madd     $ac1,                 %[result1],     %[cospi_16_64]  \n\t"
        "madd     $ac2,                 %[result2],     %[cospi_16_64]  \n\t"
        "extp     %[step2_0],           $ac1,           31              \n\t"
        "extp     %[step2_1],           $ac2,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"
        "madd     $ac3,                 %[load3],       %[cospi_24_64]  \n\t"
        "msub     $ac3,                 %[load4],       %[cospi_8_64]   \n\t"
        "extp     %[step2_2],           $ac3,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "madd     $ac1,                 %[load3],       %[cospi_8_64]   \n\t"
        "madd     $ac1,                 %[load4],       %[cospi_24_64]  \n\t"
        "extp     %[step2_3],           $ac1,           31              \n\t"

        "add      %[step1_0],           %[step2_0],     %[step2_3]      \n\t"
        "add      %[step1_1],           %[step2_1],     %[step2_2]      \n\t"
        "sub      %[step1_2],           %[step2_1],     %[step2_2]      \n\t"
        "sub      %[step1_3],           %[step2_0],     %[step2_3]      \n\t"

        : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3),
          [load4] "=&r"(load4), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [step2_0] "=&r"(step2_0),
          [step2_1] "=&r"(step2_1), [step2_2] "=&r"(step2_2),
          [step2_3] "=&r"(step2_3), [step1_0] "=r"(step1_0),
          [step1_1] "=r"(step1_1), [step1_2] "=r"(step1_2),
          [step1_3] "=r"(step1_3)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64),
          [cospi_16_64] "r"(cospi_16_64));

    __asm__ __volatile__(
        "lh       %[load5],             2(%[input])                     \n\t"
        "lh       %[load6],             30(%[input])                    \n\t"
        "lh       %[load7],             18(%[input])                    \n\t"
        "lh       %[load8],             14(%[input])                    \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "madd     $ac1,                 %[load5],       %[cospi_30_64]  \n\t"
        "msub     $ac1,                 %[load6],       %[cospi_2_64]   \n\t"
        "extp     %[result1],           $ac1,           31              \n\t"

        "madd     $ac3,                 %[load7],       %[cospi_14_64]  \n\t"
        "msub     $ac3,                 %[load8],       %[cospi_18_64]  \n\t"
        "extp     %[result2],           $ac3,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"

        "madd     $ac1,                 %[load7],       %[cospi_18_64]  \n\t"
        "madd     $ac1,                 %[load8],       %[cospi_14_64]  \n\t"
        "extp     %[result3],           $ac1,           31              \n\t"

        "madd     $ac2,                 %[load5],       %[cospi_2_64]   \n\t"
        "madd     $ac2,                 %[load6],       %[cospi_30_64]  \n\t"
        "extp     %[result4],           $ac2,           31              \n\t"

        "sub      %[load5],             %[result1],     %[result2]      \n\t"
        "sub      %[load6],             %[result4],     %[result3]      \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "madd     $ac1,                 %[load6],       %[cospi_24_64]  \n\t"
        "msub     $ac1,                 %[load5],       %[cospi_8_64]   \n\t"
        "madd     $ac3,                 %[load5],       %[cospi_24_64]  \n\t"
        "madd     $ac3,                 %[load6],       %[cospi_8_64]   \n\t"

        "extp     %[step2_9],           $ac1,           31              \n\t"
        "extp     %[step2_14],          $ac3,           31              \n\t"
        "add      %[step2_8],           %[result1],     %[result2]      \n\t"
        "add      %[step2_15],          %[result4],     %[result3]      \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7),
          [load8] "=&r"(load8), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [result3] "=&r"(result3),
          [result4] "=&r"(result4), [step2_8] "=r"(step2_8),
          [step2_15] "=r"(step2_15), [step2_9] "=r"(step2_9),
          [step2_14] "=r"(step2_14)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_30_64] "r"(cospi_30_64), [cospi_2_64] "r"(cospi_2_64),
          [cospi_14_64] "r"(cospi_14_64), [cospi_18_64] "r"(cospi_18_64),
          [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64));

    __asm__ __volatile__(
        "lh       %[load1],             10(%[input])                    \n\t"
        "lh       %[load2],             22(%[input])                    \n\t"
        "lh       %[load3],             26(%[input])                    \n\t"
        "lh       %[load4],             6(%[input])                     \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "madd     $ac1,                 %[load1],       %[cospi_22_64]  \n\t"
        "msub     $ac1,                 %[load2],       %[cospi_10_64]  \n\t"
        "extp     %[result1],           $ac1,           31              \n\t"

        "madd     $ac3,                 %[load3],       %[cospi_6_64]   \n\t"
        "msub     $ac3,                 %[load4],       %[cospi_26_64]  \n\t"
        "extp     %[result2],           $ac3,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"

        "madd     $ac1,                 %[load1],       %[cospi_10_64]  \n\t"
        "madd     $ac1,                 %[load2],       %[cospi_22_64]  \n\t"
        "extp     %[result3],           $ac1,           31              \n\t"

        "madd     $ac2,                 %[load3],       %[cospi_26_64]  \n\t"
        "madd     $ac2,                 %[load4],       %[cospi_6_64]   \n\t"
        "extp     %[result4],           $ac2,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "sub      %[load1],             %[result2],     %[result1]      \n\t"
        "sub      %[load2],             %[result4],     %[result3]      \n\t"

        "msub     $ac1,                 %[load1],       %[cospi_24_64]  \n\t"
        "msub     $ac1,                 %[load2],       %[cospi_8_64]   \n\t"
        "madd     $ac3,                 %[load2],       %[cospi_24_64]  \n\t"
        "msub     $ac3,                 %[load1],       %[cospi_8_64]   \n\t"

        "extp     %[step2_10],          $ac1,           31              \n\t"
        "extp     %[step2_13],          $ac3,           31              \n\t"
        "add      %[step2_11],          %[result1],     %[result2]      \n\t"
        "add      %[step2_12],          %[result4],     %[result3]      \n\t"

        : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3),
          [load4] "=&r"(load4), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [result3] "=&r"(result3),
          [result4] "=&r"(result4), [step2_10] "=r"(step2_10),
          [step2_11] "=r"(step2_11), [step2_12] "=r"(step2_12),
          [step2_13] "=r"(step2_13)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_22_64] "r"(cospi_22_64), [cospi_10_64] "r"(cospi_10_64),
          [cospi_6_64] "r"(cospi_6_64), [cospi_26_64] "r"(cospi_26_64),
          [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64));

    __asm__ __volatile__(
        "lh       %[load5],             4(%[input])                     \n\t"
        "lh       %[load6],             28(%[input])                    \n\t"
        "lh       %[load7],             20(%[input])                    \n\t"
        "lh       %[load8],             12(%[input])                    \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "madd     $ac1,                 %[load5],       %[cospi_28_64]  \n\t"
        "msub     $ac1,                 %[load6],       %[cospi_4_64]   \n\t"
        "extp     %[result1],           $ac1,           31              \n\t"

        "madd     $ac3,                 %[load7],       %[cospi_12_64]  \n\t"
        "msub     $ac3,                 %[load8],       %[cospi_20_64]  \n\t"
        "extp     %[result2],           $ac3,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"

        "madd     $ac1,                 %[load7],       %[cospi_20_64]  \n\t"
        "madd     $ac1,                 %[load8],       %[cospi_12_64]  \n\t"
        "extp     %[result3],           $ac1,           31              \n\t"

        "madd     $ac2,                 %[load5],       %[cospi_4_64]   \n\t"
        "madd     $ac2,                 %[load6],       %[cospi_28_64]  \n\t"
        "extp     %[result4],           $ac2,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "sub      %[load5],             %[result4],     %[result3]      \n\t"
        "sub      %[load5],             %[load5],       %[result1]      \n\t"
        "add      %[load5],             %[load5],       %[result2]      \n\t"

        "sub      %[load6],             %[result1],     %[result2]      \n\t"
        "sub      %[load6],             %[load6],       %[result3]      \n\t"
        "add      %[load6],             %[load6],       %[result4]      \n\t"

        "madd     $ac1,                 %[load5],       %[cospi_16_64]  \n\t"
        "madd     $ac3,                 %[load6],       %[cospi_16_64]  \n\t"

        "extp     %[step1_5],           $ac1,           31              \n\t"
        "extp     %[step1_6],           $ac3,           31              \n\t"
        "add      %[step1_4],           %[result1],     %[result2]      \n\t"
        "add      %[step1_7],           %[result4],     %[result3]      \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7),
          [load8] "=&r"(load8), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [result3] "=&r"(result3),
          [result4] "=&r"(result4), [step1_4] "=r"(step1_4),
          [step1_5] "=r"(step1_5), [step1_6] "=r"(step1_6),
          [step1_7] "=r"(step1_7)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_20_64] "r"(cospi_20_64), [cospi_12_64] "r"(cospi_12_64),
          [cospi_4_64] "r"(cospi_4_64), [cospi_28_64] "r"(cospi_28_64),
          [cospi_16_64] "r"(cospi_16_64));

    __asm__ __volatile__(
        "mtlo     %[const_2_power_13],  $ac0                            \n\t"
        "mthi     $zero,                $ac0                            \n\t"
        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"

        "sub      %[load5],             %[step2_14],    %[step2_13]     \n\t"
        "sub      %[load5],             %[load5],       %[step2_9]      \n\t"
        "add      %[load5],             %[load5],       %[step2_10]     \n\t"

        "madd     $ac0,                 %[load5],       %[cospi_16_64]  \n\t"

        "sub      %[load6],             %[step2_14],    %[step2_13]     \n\t"
        "sub      %[load6],             %[load6],       %[step2_10]     \n\t"
        "add      %[load6],             %[load6],       %[step2_9]      \n\t"

        "madd     $ac1,                 %[load6],       %[cospi_16_64]  \n\t"

        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "sub      %[load5],             %[step2_15],    %[step2_12]     \n\t"
        "sub      %[load5],             %[load5],       %[step2_8]      \n\t"
        "add      %[load5],             %[load5],       %[step2_11]     \n\t"

        "madd     $ac2,                 %[load5],       %[cospi_16_64]  \n\t"

        "sub      %[load6],             %[step2_15],    %[step2_12]     \n\t"
        "sub      %[load6],             %[load6],       %[step2_11]     \n\t"
        "add      %[load6],             %[load6],       %[step2_8]      \n\t"

        "madd     $ac3,                 %[load6],       %[cospi_16_64]  \n\t"

        "extp     %[step1_10],          $ac0,           31              \n\t"
        "extp     %[step1_13],          $ac1,           31              \n\t"
        "extp     %[step1_11],          $ac2,           31              \n\t"
        "extp     %[step1_12],          $ac3,           31              \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6), [step1_10] "=r"(step1_10),
          [step1_11] "=r"(step1_11), [step1_12] "=r"(step1_12),
          [step1_13] "=r"(step1_13)
        : [const_2_power_13] "r"(const_2_power_13), [step2_14] "r"(step2_14),
          [step2_13] "r"(step2_13), [step2_9] "r"(step2_9),
          [step2_10] "r"(step2_10), [step2_15] "r"(step2_15),
          [step2_12] "r"(step2_12), [step2_8] "r"(step2_8),
          [step2_11] "r"(step2_11), [cospi_16_64] "r"(cospi_16_64));

    __asm__ __volatile__(
        "add      %[load5],             %[step1_0],     %[step1_7]      \n\t"
        "add      %[load5],             %[load5],       %[step2_12]     \n\t"
        "add      %[load5],             %[load5],       %[step2_15]     \n\t"
        "add      %[load6],             %[step1_1],     %[step1_6]      \n\t"
        "add      %[load6],             %[load6],       %[step2_13]     \n\t"
        "add      %[load6],             %[load6],       %[step2_14]     \n\t"
        "sh       %[load5],             0(%[output])                    \n\t"
        "sh       %[load6],             32(%[output])                   \n\t"
        "sub      %[load5],             %[step1_1],     %[step1_6]      \n\t"
        "add      %[load5],             %[load5],       %[step2_9]      \n\t"
        "add      %[load5],             %[load5],       %[step2_10]     \n\t"
        "sub      %[load6],             %[step1_0],     %[step1_7]      \n\t"
        "add      %[load6],             %[load6],       %[step2_8]      \n\t"
        "add      %[load6],             %[load6],       %[step2_11]     \n\t"
        "sh       %[load5],             192(%[output])                  \n\t"
        "sh       %[load6],             224(%[output])                  \n\t"
        "sub      %[load5],             %[step1_0],     %[step1_7]      \n\t"
        "sub      %[load5],             %[load5],       %[step2_8]      \n\t"
        "sub      %[load5],             %[load5],       %[step2_11]     \n\t"
        "sub      %[load6],             %[step1_1],     %[step1_6]      \n\t"
        "sub      %[load6],             %[load6],       %[step2_9]      \n\t"
        "sub      %[load6],             %[load6],       %[step2_10]     \n\t"
        "sh       %[load5],             256(%[output])                  \n\t"
        "sh       %[load6],             288(%[output])                  \n\t"
        "add      %[load5],             %[step1_1],     %[step1_6]      \n\t"
        "sub      %[load5],             %[load5],       %[step2_13]     \n\t"
        "sub      %[load5],             %[load5],       %[step2_14]     \n\t"
        "add      %[load6],             %[step1_0],     %[step1_7]      \n\t"
        "sub      %[load6],             %[load6],       %[step2_12]     \n\t"
        "sub      %[load6],             %[load6],       %[step2_15]     \n\t"
        "sh       %[load5],             448(%[output])                  \n\t"
        "sh       %[load6],             480(%[output])                  \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6)
        : [output] "r"(output), [step1_0] "r"(step1_0), [step1_1] "r"(step1_1),
          [step1_6] "r"(step1_6), [step1_7] "r"(step1_7),
          [step2_8] "r"(step2_8), [step2_9] "r"(step2_9),
          [step2_10] "r"(step2_10), [step2_11] "r"(step2_11),
          [step2_12] "r"(step2_12), [step2_13] "r"(step2_13),
          [step2_14] "r"(step2_14), [step2_15] "r"(step2_15));

    __asm__ __volatile__(
        "add      %[load5],             %[step1_2],     %[step1_5]      \n\t"
        "add      %[load5],             %[load5],       %[step1_13]     \n\t"
        "add      %[load6],             %[step1_3],     %[step1_4]      \n\t"
        "add      %[load6],             %[load6],       %[step1_12]     \n\t"
        "sh       %[load5],             64(%[output])                   \n\t"
        "sh       %[load6],             96(%[output])                   \n\t"
        "sub      %[load5],             %[step1_3],     %[step1_4]      \n\t"
        "add      %[load5],             %[load5],       %[step1_11]     \n\t"
        "sub      %[load6],             %[step1_2],     %[step1_5]      \n\t"
        "add      %[load6],             %[load6],       %[step1_10]     \n\t"
        "sh       %[load5],             128(%[output])                  \n\t"
        "sh       %[load6],             160(%[output])                  \n\t"
        "sub      %[load5],             %[step1_2],     %[step1_5]      \n\t"
        "sub      %[load5],             %[load5],       %[step1_10]     \n\t"
        "sub      %[load6],             %[step1_3],     %[step1_4]      \n\t"
        "sub      %[load6],             %[load6],       %[step1_11]     \n\t"
        "sh       %[load5],             320(%[output])                  \n\t"
        "sh       %[load6],             352(%[output])                  \n\t"
        "add      %[load5],             %[step1_3],     %[step1_4]      \n\t"
        "sub      %[load5],             %[load5],       %[step1_12]     \n\t"
        "add      %[load6],             %[step1_2],     %[step1_5]      \n\t"
        "sub      %[load6],             %[load6],       %[step1_13]     \n\t"
        "sh       %[load5],             384(%[output])                  \n\t"
        "sh       %[load6],             416(%[output])                  \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6)
        : [output] "r"(output), [step1_2] "r"(step1_2), [step1_3] "r"(step1_3),
          [step1_4] "r"(step1_4), [step1_5] "r"(step1_5),
          [step1_10] "r"(step1_10), [step1_11] "r"(step1_11),
          [step1_12] "r"(step1_12), [step1_13] "r"(step1_13));

    input += 16;
    output += 1;
  }
}

void idct16_cols_add_blk_dspr2(int16_t *input, uint8_t *dest, int dest_stride) {
  int i;
  int step1_0, step1_1, step1_2, step1_3, step1_4, step1_5, step1_6, step1_7;
  int step1_8, step1_9, step1_10, step1_11;
  int step1_12, step1_13, step1_14, step1_15;
  int step2_0, step2_1, step2_2, step2_3;
  int step2_8, step2_9, step2_10, step2_11;
  int step2_12, step2_13, step2_14, step2_15;
  int load1, load2, load3, load4, load5, load6, load7, load8;
  int result1, result2, result3, result4;
  const int const_2_power_13 = 8192;
  uint8_t *dest_pix;
  uint8_t *cm = aom_ff_cropTbl;

  /* prefetch aom_ff_cropTbl */
  prefetch_load(aom_ff_cropTbl);
  prefetch_load(aom_ff_cropTbl + 32);
  prefetch_load(aom_ff_cropTbl + 64);
  prefetch_load(aom_ff_cropTbl + 96);
  prefetch_load(aom_ff_cropTbl + 128);
  prefetch_load(aom_ff_cropTbl + 160);
  prefetch_load(aom_ff_cropTbl + 192);
  prefetch_load(aom_ff_cropTbl + 224);

  for (i = 0; i < 16; ++i) {
    dest_pix = (dest + i);
    __asm__ __volatile__(
        "lh       %[load1],              0(%[input])                    \n\t"
        "lh       %[load2],             16(%[input])                    \n\t"
        "lh       %[load3],              8(%[input])                    \n\t"
        "lh       %[load4],             24(%[input])                    \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"
        "add      %[result1],           %[load1],       %[load2]        \n\t"
        "sub      %[result2],           %[load1],       %[load2]        \n\t"
        "madd     $ac1,                 %[result1],     %[cospi_16_64]  \n\t"
        "madd     $ac2,                 %[result2],     %[cospi_16_64]  \n\t"
        "extp     %[step2_0],           $ac1,           31              \n\t"
        "extp     %[step2_1],           $ac2,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"
        "madd     $ac3,                 %[load3],       %[cospi_24_64]  \n\t"
        "msub     $ac3,                 %[load4],       %[cospi_8_64]   \n\t"
        "extp     %[step2_2],           $ac3,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "madd     $ac1,                 %[load3],       %[cospi_8_64]   \n\t"
        "madd     $ac1,                 %[load4],       %[cospi_24_64]  \n\t"
        "extp     %[step2_3],           $ac1,           31              \n\t"

        "add      %[step1_0],           %[step2_0],     %[step2_3]      \n\t"
        "add      %[step1_1],           %[step2_1],     %[step2_2]      \n\t"
        "sub      %[step1_2],           %[step2_1],     %[step2_2]      \n\t"
        "sub      %[step1_3],           %[step2_0],     %[step2_3]      \n\t"

        : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3),
          [load4] "=&r"(load4), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [step2_0] "=&r"(step2_0),
          [step2_1] "=&r"(step2_1), [step2_2] "=&r"(step2_2),
          [step2_3] "=&r"(step2_3), [step1_0] "=r"(step1_0),
          [step1_1] "=r"(step1_1), [step1_2] "=r"(step1_2),
          [step1_3] "=r"(step1_3)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64),
          [cospi_16_64] "r"(cospi_16_64));

    __asm__ __volatile__(
        "lh       %[load5],             2(%[input])                     \n\t"
        "lh       %[load6],             30(%[input])                    \n\t"
        "lh       %[load7],             18(%[input])                    \n\t"
        "lh       %[load8],             14(%[input])                    \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "madd     $ac1,                 %[load5],       %[cospi_30_64]  \n\t"
        "msub     $ac1,                 %[load6],       %[cospi_2_64]   \n\t"
        "extp     %[result1],           $ac1,           31              \n\t"

        "madd     $ac3,                 %[load7],       %[cospi_14_64]  \n\t"
        "msub     $ac3,                 %[load8],       %[cospi_18_64]  \n\t"
        "extp     %[result2],           $ac3,           31              \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"

        "madd     $ac1,                 %[load7],       %[cospi_18_64]  \n\t"
        "madd     $ac1,                 %[load8],       %[cospi_14_64]  \n\t"
        "extp     %[result3],           $ac1,           31              \n\t"

        "madd     $ac2,                 %[load5],        %[cospi_2_64]  \n\t"
        "madd     $ac2,                 %[load6],        %[cospi_30_64] \n\t"
        "extp     %[result4],           $ac2,            31             \n\t"

        "sub      %[load5],             %[result1],     %[result2]      \n\t"
        "sub      %[load6],             %[result4],     %[result3]      \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "madd     $ac1,                 %[load6],       %[cospi_24_64]  \n\t"
        "msub     $ac1,                 %[load5],       %[cospi_8_64]   \n\t"
        "madd     $ac3,                 %[load5],       %[cospi_24_64]  \n\t"
        "madd     $ac3,                 %[load6],       %[cospi_8_64]   \n\t"

        "extp     %[step2_9],           $ac1,           31              \n\t"
        "extp     %[step2_14],          $ac3,           31              \n\t"
        "add      %[step2_8],           %[result1],     %[result2]      \n\t"
        "add      %[step2_15],          %[result4],     %[result3]      \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7),
          [load8] "=&r"(load8), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [result3] "=&r"(result3),
          [result4] "=&r"(result4), [step2_8] "=r"(step2_8),
          [step2_15] "=r"(step2_15), [step2_9] "=r"(step2_9),
          [step2_14] "=r"(step2_14)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_30_64] "r"(cospi_30_64), [cospi_2_64] "r"(cospi_2_64),
          [cospi_14_64] "r"(cospi_14_64), [cospi_18_64] "r"(cospi_18_64),
          [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64));

    __asm__ __volatile__(
        "lh       %[load1],             10(%[input])                    \n\t"
        "lh       %[load2],             22(%[input])                    \n\t"
        "lh       %[load3],             26(%[input])                    \n\t"
        "lh       %[load4],             6(%[input])                     \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "madd     $ac1,                 %[load1],    %[cospi_22_64]     \n\t"
        "msub     $ac1,                 %[load2],    %[cospi_10_64]     \n\t"
        "extp     %[result1],           $ac1,        31                 \n\t"

        "madd     $ac3,                 %[load3],    %[cospi_6_64]      \n\t"
        "msub     $ac3,                 %[load4],    %[cospi_26_64]     \n\t"
        "extp     %[result2],           $ac3,        31                 \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"

        "madd     $ac1,                 %[load1],    %[cospi_10_64]     \n\t"
        "madd     $ac1,                 %[load2],    %[cospi_22_64]     \n\t"
        "extp     %[result3],           $ac1,        31                 \n\t"

        "madd     $ac2,                 %[load3],    %[cospi_26_64]     \n\t"
        "madd     $ac2,                 %[load4],    %[cospi_6_64]      \n\t"
        "extp     %[result4],           $ac2,        31                 \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "sub      %[load1],             %[result2],     %[result1]      \n\t"
        "sub      %[load2],             %[result4],     %[result3]      \n\t"

        "msub     $ac1,                 %[load1],       %[cospi_24_64]  \n\t"
        "msub     $ac1,                 %[load2],       %[cospi_8_64]   \n\t"
        "madd     $ac3,                 %[load2],       %[cospi_24_64]  \n\t"
        "msub     $ac3,                 %[load1],       %[cospi_8_64]   \n\t"

        "extp     %[step2_10],          $ac1,           31              \n\t"
        "extp     %[step2_13],          $ac3,           31              \n\t"
        "add      %[step2_11],          %[result1],     %[result2]      \n\t"
        "add      %[step2_12],          %[result4],     %[result3]      \n\t"

        : [load1] "=&r"(load1), [load2] "=&r"(load2), [load3] "=&r"(load3),
          [load4] "=&r"(load4), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [result3] "=&r"(result3),
          [result4] "=&r"(result4), [step2_10] "=r"(step2_10),
          [step2_11] "=r"(step2_11), [step2_12] "=r"(step2_12),
          [step2_13] "=r"(step2_13)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_22_64] "r"(cospi_22_64), [cospi_10_64] "r"(cospi_10_64),
          [cospi_6_64] "r"(cospi_6_64), [cospi_26_64] "r"(cospi_26_64),
          [cospi_24_64] "r"(cospi_24_64), [cospi_8_64] "r"(cospi_8_64));

    __asm__ __volatile__(
        "lh       %[load5],             4(%[input])                   \n\t"
        "lh       %[load6],             28(%[input])                  \n\t"
        "lh       %[load7],             20(%[input])                  \n\t"
        "lh       %[load8],             12(%[input])                  \n\t"

        "mtlo     %[const_2_power_13],  $ac1                          \n\t"
        "mthi     $zero,                $ac1                          \n\t"
        "mtlo     %[const_2_power_13],  $ac3                          \n\t"
        "mthi     $zero,                $ac3                          \n\t"

        "madd     $ac1,                 %[load5],    %[cospi_28_64]   \n\t"
        "msub     $ac1,                 %[load6],    %[cospi_4_64]    \n\t"
        "extp     %[result1],           $ac1,        31               \n\t"

        "madd     $ac3,                 %[load7],    %[cospi_12_64]   \n\t"
        "msub     $ac3,                 %[load8],    %[cospi_20_64]   \n\t"
        "extp     %[result2],           $ac3,        31               \n\t"

        "mtlo     %[const_2_power_13],  $ac1                          \n\t"
        "mthi     $zero,                $ac1                          \n\t"
        "mtlo     %[const_2_power_13],  $ac2                          \n\t"
        "mthi     $zero,                $ac2                          \n\t"

        "madd     $ac1,                 %[load7],    %[cospi_20_64]   \n\t"
        "madd     $ac1,                 %[load8],    %[cospi_12_64]   \n\t"
        "extp     %[result3],           $ac1,        31               \n\t"

        "madd     $ac2,                 %[load5],    %[cospi_4_64]    \n\t"
        "madd     $ac2,                 %[load6],    %[cospi_28_64]   \n\t"
        "extp     %[result4],           $ac2,        31               \n\t"

        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "sub      %[load5],             %[result4],     %[result3]      \n\t"
        "sub      %[load5],             %[load5],       %[result1]      \n\t"
        "add      %[load5],             %[load5],       %[result2]      \n\t"

        "sub      %[load6],             %[result1],     %[result2]      \n\t"
        "sub      %[load6],             %[load6],       %[result3]      \n\t"
        "add      %[load6],             %[load6],       %[result4]      \n\t"

        "madd     $ac1,                 %[load5],       %[cospi_16_64]  \n\t"
        "madd     $ac3,                 %[load6],       %[cospi_16_64]  \n\t"

        "extp     %[step1_5],           $ac1,           31              \n\t"
        "extp     %[step1_6],           $ac3,           31              \n\t"

        "add      %[step1_4],           %[result1],     %[result2]      \n\t"
        "add      %[step1_7],           %[result4],     %[result3]      \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7),
          [load8] "=&r"(load8), [result1] "=&r"(result1),
          [result2] "=&r"(result2), [result3] "=&r"(result3),
          [result4] "=&r"(result4), [step1_4] "=r"(step1_4),
          [step1_5] "=r"(step1_5), [step1_6] "=r"(step1_6),
          [step1_7] "=r"(step1_7)
        : [const_2_power_13] "r"(const_2_power_13), [input] "r"(input),
          [cospi_20_64] "r"(cospi_20_64), [cospi_12_64] "r"(cospi_12_64),
          [cospi_4_64] "r"(cospi_4_64), [cospi_28_64] "r"(cospi_28_64),
          [cospi_16_64] "r"(cospi_16_64));

    __asm__ __volatile__(
        "mtlo     %[const_2_power_13],  $ac0                            \n\t"
        "mthi     $zero,                $ac0                            \n\t"
        "mtlo     %[const_2_power_13],  $ac1                            \n\t"
        "mthi     $zero,                $ac1                            \n\t"

        "sub      %[load5],             %[step2_14],    %[step2_13]     \n\t"
        "sub      %[load5],             %[load5],       %[step2_9]      \n\t"
        "add      %[load5],             %[load5],       %[step2_10]     \n\t"

        "madd     $ac0,                 %[load5],       %[cospi_16_64]  \n\t"

        "sub      %[load6],             %[step2_14],    %[step2_13]     \n\t"
        "sub      %[load6],             %[load6],       %[step2_10]     \n\t"
        "add      %[load6],             %[load6],       %[step2_9]      \n\t"

        "madd     $ac1,                 %[load6],       %[cospi_16_64]  \n\t"

        "mtlo     %[const_2_power_13],  $ac2                            \n\t"
        "mthi     $zero,                $ac2                            \n\t"
        "mtlo     %[const_2_power_13],  $ac3                            \n\t"
        "mthi     $zero,                $ac3                            \n\t"

        "sub      %[load5],             %[step2_15],    %[step2_12]     \n\t"
        "sub      %[load5],             %[load5],       %[step2_8]      \n\t"
        "add      %[load5],             %[load5],       %[step2_11]     \n\t"

        "madd     $ac2,                 %[load5],       %[cospi_16_64]  \n\t"

        "sub      %[load6],             %[step2_15],    %[step2_12]     \n\t"
        "sub      %[load6],             %[load6],       %[step2_11]     \n\t"
        "add      %[load6],             %[load6],       %[step2_8]      \n\t"

        "madd     $ac3,                 %[load6],       %[cospi_16_64]  \n\t"

        "extp     %[step1_10],          $ac0,           31              \n\t"
        "extp     %[step1_13],          $ac1,           31              \n\t"
        "extp     %[step1_11],          $ac2,           31              \n\t"
        "extp     %[step1_12],          $ac3,           31              \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6), [step1_10] "=r"(step1_10),
          [step1_11] "=r"(step1_11), [step1_12] "=r"(step1_12),
          [step1_13] "=r"(step1_13)
        : [const_2_power_13] "r"(const_2_power_13), [step2_14] "r"(step2_14),
          [step2_13] "r"(step2_13), [step2_9] "r"(step2_9),
          [step2_10] "r"(step2_10), [step2_15] "r"(step2_15),
          [step2_12] "r"(step2_12), [step2_8] "r"(step2_8),
          [step2_11] "r"(step2_11), [cospi_16_64] "r"(cospi_16_64));

    step1_8 = step2_8 + step2_11;
    step1_9 = step2_9 + step2_10;
    step1_14 = step2_13 + step2_14;
    step1_15 = step2_12 + step2_15;

    __asm__ __volatile__(
        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "add      %[load5],         %[step1_0],         %[step1_7]      \n\t"
        "add      %[load5],         %[load5],           %[step1_15]     \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "add      %[load6],         %[step1_1],         %[step1_6]      \n\t"
        "add      %[load6],         %[load6],           %[step1_14]     \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"

        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "add      %[load5],         %[step1_2],         %[step1_5]      \n\t"
        "add      %[load5],         %[load5],           %[step1_13]     \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "add      %[load6],         %[step1_3],         %[step1_4]      \n\t"
        "add      %[load6],         %[load6],           %[step1_12]     \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"

        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "sub      %[load5],         %[step1_3],         %[step1_4]      \n\t"
        "add      %[load5],         %[load5],           %[step1_11]     \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "sub      %[load6],         %[step1_2],         %[step1_5]      \n\t"
        "add      %[load6],         %[load6],           %[step1_10]     \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"

        "sub      %[load5],         %[step1_1],         %[step1_6]      \n\t"
        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "add      %[load5],         %[load5],           %[step1_9]      \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "sub      %[load6],         %[step1_0],         %[step1_7]      \n\t"
        "add      %[load6],         %[load6],           %[step1_8]      \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"

        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "sub      %[load5],         %[step1_0],         %[step1_7]      \n\t"
        "sub      %[load5],         %[load5],           %[step1_8]      \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "sub      %[load6],         %[step1_1],         %[step1_6]      \n\t"
        "sub      %[load6],         %[load6],           %[step1_9]      \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"

        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "sub      %[load5],         %[step1_2],         %[step1_5]      \n\t"
        "sub      %[load5],         %[load5],           %[step1_10]     \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "sub      %[load6],         %[step1_3],         %[step1_4]      \n\t"
        "sub      %[load6],         %[load6],           %[step1_11]     \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"

        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "add      %[load5],         %[step1_3],         %[step1_4]      \n\t"
        "sub      %[load5],         %[load5],           %[step1_12]     \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "add      %[load6],         %[step1_2],         %[step1_5]      \n\t"
        "sub      %[load6],         %[load6],           %[step1_13]     \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"

        "lbu      %[load7],         0(%[dest_pix])                      \n\t"
        "add      %[load5],         %[step1_1],         %[step1_6]      \n\t"
        "sub      %[load5],         %[load5],           %[step1_14]     \n\t"
        "addi     %[load5],         %[load5],           32              \n\t"
        "sra      %[load5],         %[load5],           6               \n\t"
        "add      %[load7],         %[load7],           %[load5]        \n\t"
        "lbux     %[load5],         %[load7](%[cm])                     \n\t"
        "add      %[load6],         %[step1_0],         %[step1_7]      \n\t"
        "sub      %[load6],         %[load6],           %[step1_15]     \n\t"
        "sb       %[load5],         0(%[dest_pix])                      \n\t"
        "addu     %[dest_pix],      %[dest_pix],        %[dest_stride]  \n\t"
        "lbu      %[load8],         0(%[dest_pix])                      \n\t"
        "addi     %[load6],         %[load6],           32              \n\t"
        "sra      %[load6],         %[load6],           6               \n\t"
        "add      %[load8],         %[load8],           %[load6]        \n\t"
        "lbux     %[load6],         %[load8](%[cm])                     \n\t"
        "sb       %[load6],         0(%[dest_pix])                      \n\t"

        : [load5] "=&r"(load5), [load6] "=&r"(load6), [load7] "=&r"(load7),
          [load8] "=&r"(load8), [dest_pix] "+r"(dest_pix)
        :
        [cm] "r"(cm), [dest_stride] "r"(dest_stride), [step1_0] "r"(step1_0),
        [step1_1] "r"(step1_1), [step1_2] "r"(step1_2), [step1_3] "r"(step1_3),
        [step1_4] "r"(step1_4), [step1_5] "r"(step1_5), [step1_6] "r"(step1_6),
        [step1_7] "r"(step1_7), [step1_8] "r"(step1_8), [step1_9] "r"(step1_9),
        [step1_10] "r"(step1_10), [step1_11] "r"(step1_11),
        [step1_12] "r"(step1_12), [step1_13] "r"(step1_13),
        [step1_14] "r"(step1_14), [step1_15] "r"(step1_15));

    input += 16;
  }
}

void aom_idct16x16_256_add_dspr2(const int16_t *input, uint8_t *dest,
                                 int dest_stride) {
  DECLARE_ALIGNED(32, int16_t, out[16 * 16]);
  uint32_t pos = 45;

  /* bit positon for extract from acc */
  __asm__ __volatile__("wrdsp    %[pos],    1    \n\t" : : [pos] "r"(pos));

  // First transform rows
  idct16_rows_dspr2(input, out, 16);

  // Then transform columns and add to dest
  idct16_cols_add_blk_dspr2(out, dest, dest_stride);
}

void aom_idct16x16_10_add_dspr2(const int16_t *input, uint8_t *dest,
                                int dest_stride) {
  DECLARE_ALIGNED(32, int16_t, out[16 * 16]);
  int16_t *outptr = out;
  uint32_t i;
  uint32_t pos = 45;

  /* bit positon for extract from acc */
  __asm__ __volatile__("wrdsp    %[pos],    1    \n\t" : : [pos] "r"(pos));

  // First transform rows. Since all non-zero dct coefficients are in
  // upper-left 4x4 area, we only need to calculate first 4 rows here.
  idct16_rows_dspr2(input, outptr, 4);

  outptr += 4;
  for (i = 0; i < 6; ++i) {
    __asm__ __volatile__(
        "sw     $zero,    0(%[outptr])     \n\t"
        "sw     $zero,   32(%[outptr])     \n\t"
        "sw     $zero,   64(%[outptr])     \n\t"
        "sw     $zero,   96(%[outptr])     \n\t"
        "sw     $zero,  128(%[outptr])     \n\t"
        "sw     $zero,  160(%[outptr])     \n\t"
        "sw     $zero,  192(%[outptr])     \n\t"
        "sw     $zero,  224(%[outptr])     \n\t"
        "sw     $zero,  256(%[outptr])     \n\t"
        "sw     $zero,  288(%[outptr])     \n\t"
        "sw     $zero,  320(%[outptr])     \n\t"
        "sw     $zero,  352(%[outptr])     \n\t"
        "sw     $zero,  384(%[outptr])     \n\t"
        "sw     $zero,  416(%[outptr])     \n\t"
        "sw     $zero,  448(%[outptr])     \n\t"
        "sw     $zero,  480(%[outptr])     \n\t"

        :
        : [outptr] "r"(outptr));

    outptr += 2;
  }

  // Then transform columns
  idct16_cols_add_blk_dspr2(out, dest, dest_stride);
}

void aom_idct16x16_1_add_dspr2(const int16_t *input, uint8_t *dest,
                               int dest_stride) {
  uint32_t pos = 45;
  int32_t out;
  int32_t r;
  int32_t a1, absa1;
  int32_t vector_a1;
  int32_t t1, t2, t3, t4;
  int32_t vector_1, vector_2, vector_3, vector_4;

  /* bit positon for extract from acc */
  __asm__ __volatile__("wrdsp      %[pos],     1           \n\t"

                       :
                       : [pos] "r"(pos));

  out = DCT_CONST_ROUND_SHIFT_TWICE_COSPI_16_64(input[0]);
  __asm__ __volatile__(
      "addi     %[out],     %[out],     32      \n\t"
      "sra      %[a1],      %[out],     6       \n\t"

      : [out] "+r"(out), [a1] "=r"(a1)
      :);

  if (a1 < 0) {
    /* use quad-byte
     * input and output memory are four byte aligned */
    __asm__ __volatile__(
        "abs        %[absa1],       %[a1]       \n\t"
        "replv.qb   %[vector_a1],   %[absa1]    \n\t"

        : [absa1] "=r"(absa1), [vector_a1] "=r"(vector_a1)
        : [a1] "r"(a1));

    for (r = 16; r--;) {
      __asm__ __volatile__(
          "lw             %[t1],          0(%[dest])                      \n\t"
          "lw             %[t2],          4(%[dest])                      \n\t"
          "lw             %[t3],          8(%[dest])                      \n\t"
          "lw             %[t4],          12(%[dest])                     \n\t"
          "subu_s.qb      %[vector_1],    %[t1],          %[vector_a1]    \n\t"
          "subu_s.qb      %[vector_2],    %[t2],          %[vector_a1]    \n\t"
          "subu_s.qb      %[vector_3],    %[t3],          %[vector_a1]    \n\t"
          "subu_s.qb      %[vector_4],    %[t4],          %[vector_a1]    \n\t"
          "sw             %[vector_1],    0(%[dest])                      \n\t"
          "sw             %[vector_2],    4(%[dest])                      \n\t"
          "sw             %[vector_3],    8(%[dest])                      \n\t"
          "sw             %[vector_4],    12(%[dest])                     \n\t"
          "add            %[dest],        %[dest],        %[dest_stride]  \n\t"

          : [t1] "=&r"(t1), [t2] "=&r"(t2), [t3] "=&r"(t3), [t4] "=&r"(t4),
            [vector_1] "=&r"(vector_1), [vector_2] "=&r"(vector_2),
            [vector_3] "=&r"(vector_3), [vector_4] "=&r"(vector_4),
            [dest] "+&r"(dest)
          : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1));
    }
  } else {
    /* use quad-byte
     * input and output memory are four byte aligned */
    __asm__ __volatile__("replv.qb   %[vector_a1],   %[a1]   \n\t"

                         : [vector_a1] "=r"(vector_a1)
                         : [a1] "r"(a1));

    for (r = 16; r--;) {
      __asm__ __volatile__(
          "lw             %[t1],          0(%[dest])                      \n\t"
          "lw             %[t2],          4(%[dest])                      \n\t"
          "lw             %[t3],          8(%[dest])                      \n\t"
          "lw             %[t4],          12(%[dest])                     \n\t"
          "addu_s.qb      %[vector_1],    %[t1],          %[vector_a1]    \n\t"
          "addu_s.qb      %[vector_2],    %[t2],          %[vector_a1]    \n\t"
          "addu_s.qb      %[vector_3],    %[t3],          %[vector_a1]    \n\t"
          "addu_s.qb      %[vector_4],    %[t4],          %[vector_a1]    \n\t"
          "sw             %[vector_1],    0(%[dest])                      \n\t"
          "sw             %[vector_2],    4(%[dest])                      \n\t"
          "sw             %[vector_3],    8(%[dest])                      \n\t"
          "sw             %[vector_4],    12(%[dest])                     \n\t"
          "add            %[dest],        %[dest],        %[dest_stride]  \n\t"

          : [t1] "=&r"(t1), [t2] "=&r"(t2), [t3] "=&r"(t3), [t4] "=&r"(t4),
            [vector_1] "=&r"(vector_1), [vector_2] "=&r"(vector_2),
            [vector_3] "=&r"(vector_3), [vector_4] "=&r"(vector_4),
            [dest] "+&r"(dest)
          : [dest_stride] "r"(dest_stride), [vector_a1] "r"(vector_a1));
    }
  }
}

void iadst16_dspr2(const int16_t *input, int16_t *output) {
  int s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15;

  int x0 = input[15];
  int x1 = input[0];
  int x2 = input[13];
  int x3 = input[2];
  int x4 = input[11];
  int x5 = input[4];
  int x6 = input[9];
  int x7 = input[6];
  int x8 = input[7];
  int x9 = input[8];
  int x10 = input[5];
  int x11 = input[10];
  int x12 = input[3];
  int x13 = input[12];
  int x14 = input[1];
  int x15 = input[14];

  if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 |
        x13 | x14 | x15)) {
    output[0] = output[1] = output[2] = output[3] = output[4] = output[5] =
        output[6] = output[7] = output[8] = output[9] = output[10] =
            output[11] = output[12] = output[13] = output[14] = output[15] = 0;
    return;
  }

  // stage 1
  s0 = x0 * cospi_1_64 + x1 * cospi_31_64;
  s1 = x0 * cospi_31_64 - x1 * cospi_1_64;
  s2 = x2 * cospi_5_64 + x3 * cospi_27_64;
  s3 = x2 * cospi_27_64 - x3 * cospi_5_64;
  s4 = x4 * cospi_9_64 + x5 * cospi_23_64;
  s5 = x4 * cospi_23_64 - x5 * cospi_9_64;
  s6 = x6 * cospi_13_64 + x7 * cospi_19_64;
  s7 = x6 * cospi_19_64 - x7 * cospi_13_64;
  s8 = x8 * cospi_17_64 + x9 * cospi_15_64;
  s9 = x8 * cospi_15_64 - x9 * cospi_17_64;
  s10 = x10 * cospi_21_64 + x11 * cospi_11_64;
  s11 = x10 * cospi_11_64 - x11 * cospi_21_64;
  s12 = x12 * cospi_25_64 + x13 * cospi_7_64;
  s13 = x12 * cospi_7_64 - x13 * cospi_25_64;
  s14 = x14 * cospi_29_64 + x15 * cospi_3_64;
  s15 = x14 * cospi_3_64 - x15 * cospi_29_64;

  x0 = dct_const_round_shift(s0 + s8);
  x1 = dct_const_round_shift(s1 + s9);
  x2 = dct_const_round_shift(s2 + s10);
  x3 = dct_const_round_shift(s3 + s11);
  x4 = dct_const_round_shift(s4 + s12);
  x5 = dct_const_round_shift(s5 + s13);
  x6 = dct_const_round_shift(s6 + s14);
  x7 = dct_const_round_shift(s7 + s15);
  x8 = dct_const_round_shift(s0 - s8);
  x9 = dct_const_round_shift(s1 - s9);
  x10 = dct_const_round_shift(s2 - s10);
  x11 = dct_const_round_shift(s3 - s11);
  x12 = dct_const_round_shift(s4 - s12);
  x13 = dct_const_round_shift(s5 - s13);
  x14 = dct_const_round_shift(s6 - s14);
  x15 = dct_const_round_shift(s7 - s15);

  // stage 2
  s0 = x0;
  s1 = x1;
  s2 = x2;
  s3 = x3;
  s4 = x4;
  s5 = x5;
  s6 = x6;
  s7 = x7;
  s8 = x8 * cospi_4_64 + x9 * cospi_28_64;
  s9 = x8 * cospi_28_64 - x9 * cospi_4_64;
  s10 = x10 * cospi_20_64 + x11 * cospi_12_64;
  s11 = x10 * cospi_12_64 - x11 * cospi_20_64;
  s12 = -x12 * cospi_28_64 + x13 * cospi_4_64;
  s13 = x12 * cospi_4_64 + x13 * cospi_28_64;
  s14 = -x14 * cospi_12_64 + x15 * cospi_20_64;
  s15 = x14 * cospi_20_64 + x15 * cospi_12_64;

  x0 = s0 + s4;
  x1 = s1 + s5;
  x2 = s2 + s6;
  x3 = s3 + s7;
  x4 = s0 - s4;
  x5 = s1 - s5;
  x6 = s2 - s6;
  x7 = s3 - s7;
  x8 = dct_const_round_shift(s8 + s12);
  x9 = dct_const_round_shift(s9 + s13);
  x10 = dct_const_round_shift(s10 + s14);
  x11 = dct_const_round_shift(s11 + s15);
  x12 = dct_const_round_shift(s8 - s12);
  x13 = dct_const_round_shift(s9 - s13);
  x14 = dct_const_round_shift(s10 - s14);
  x15 = dct_const_round_shift(s11 - s15);

  // stage 3
  s0 = x0;
  s1 = x1;
  s2 = x2;
  s3 = x3;
  s4 = x4 * cospi_8_64 + x5 * cospi_24_64;
  s5 = x4 * cospi_24_64 - x5 * cospi_8_64;
  s6 = -x6 * cospi_24_64 + x7 * cospi_8_64;
  s7 = x6 * cospi_8_64 + x7 * cospi_24_64;
  s8 = x8;
  s9 = x9;
  s10 = x10;
  s11 = x11;
  s12 = x12 * cospi_8_64 + x13 * cospi_24_64;
  s13 = x12 * cospi_24_64 - x13 * cospi_8_64;
  s14 = -x14 * cospi_24_64 + x15 * cospi_8_64;
  s15 = x14 * cospi_8_64 + x15 * cospi_24_64;

  x0 = s0 + s2;
  x1 = s1 + s3;
  x2 = s0 - s2;
  x3 = s1 - s3;
  x4 = dct_const_round_shift(s4 + s6);
  x5 = dct_const_round_shift(s5 + s7);
  x6 = dct_const_round_shift(s4 - s6);
  x7 = dct_const_round_shift(s5 - s7);
  x8 = s8 + s10;
  x9 = s9 + s11;
  x10 = s8 - s10;
  x11 = s9 - s11;
  x12 = dct_const_round_shift(s12 + s14);
  x13 = dct_const_round_shift(s13 + s15);
  x14 = dct_const_round_shift(s12 - s14);
  x15 = dct_const_round_shift(s13 - s15);

  // stage 4
  s2 = (-cospi_16_64) * (x2 + x3);
  s3 = cospi_16_64 * (x2 - x3);
  s6 = cospi_16_64 * (x6 + x7);
  s7 = cospi_16_64 * (-x6 + x7);
  s10 = cospi_16_64 * (x10 + x11);
  s11 = cospi_16_64 * (-x10 + x11);
  s14 = (-cospi_16_64) * (x14 + x15);
  s15 = cospi_16_64 * (x14 - x15);

  x2 = dct_const_round_shift(s2);
  x3 = dct_const_round_shift(s3);
  x6 = dct_const_round_shift(s6);
  x7 = dct_const_round_shift(s7);
  x10 = dct_const_round_shift(s10);
  x11 = dct_const_round_shift(s11);
  x14 = dct_const_round_shift(s14);
  x15 = dct_const_round_shift(s15);

  output[0] = x0;
  output[1] = -x8;
  output[2] = x12;
  output[3] = -x4;
  output[4] = x6;
  output[5] = x14;
  output[6] = x10;
  output[7] = x2;
  output[8] = x3;
  output[9] = x11;
  output[10] = x15;
  output[11] = x7;
  output[12] = x5;
  output[13] = -x13;
  output[14] = x9;
  output[15] = -x1;
}

#endif  // HAVE_DSPR2